{ "$schema": "./torchwright_schematic_v1.schema.json", "artifact": { "architecture": "phi3", "files": { "config.json": { "bytes": 999, "sha256": "37717c2991cc4a4bf9b3ead11dc7c561a446d1d0bc664095fb209141061df64b" }, "generation_config.json": { "bytes": 105, "sha256": "6ebdfc7ea4a3b172e5baf014667c957a7eb90ea465d05178b801cc2ab7b6bdc4" }, "model-00001-of-00030.safetensors": { "bytes": 151036360, "sha256": "76b7a6fd9c3e496d29a0c31e61103fec527cd3ec04f27cef8751d53929af46e2" }, "model-00002-of-00030.safetensors": { "bytes": 151036360, "sha256": "5526d0245af6bbfe2854fb9a966718986d1ef37a369eeaa603b2050cbb0e4270" }, "model-00003-of-00030.safetensors": { "bytes": 151036360, "sha256": "5e2c7cbf2261296fb98702488a0c21e98400c783c03382ab7cbbc93c770c46ee" }, "model-00004-of-00030.safetensors": { "bytes": 151036360, "sha256": "06e7dadef23c49c64be498085f711509320d3799a16e1a5b845ce7e5b23bc757" }, "model-00005-of-00030.safetensors": { "bytes": 151036360, "sha256": "160052d72bc5e4878bd53550e79b1d5c2d118c419cd5e8155ffd5de146c5ecae" }, "model-00006-of-00030.safetensors": { "bytes": 151036360, "sha256": "790450cc87f0ec74dbb9dc1085762211b0482647c7967d9c456ba41e6d5f4e7d" }, "model-00007-of-00030.safetensors": { "bytes": 151036360, "sha256": "f852c383685754348b628ee3f2cbc302300f6b514ca327f81661f46d27128524" }, "model-00008-of-00030.safetensors": { "bytes": 151036360, "sha256": "c40046a69261438db5e9bd521ee24bfd67c503e848be778cd72cd573f4663b2e" }, "model-00009-of-00030.safetensors": { "bytes": 151036360, "sha256": "84cd6addaea796a52e115be45d7355541068b7ca02acd2d5204cc3d5a90a554c" }, "model-00010-of-00030.safetensors": { "bytes": 151036360, "sha256": "ce9561edda984e98310e0a50d9d5dea9697d8529e9a571f2654a80a736fc7c99" }, "model-00011-of-00030.safetensors": { "bytes": 151036360, "sha256": "84517d7884ccdf5ac7752ef8bb1bf21883b128b8fa5005b46d9f5da014808698" }, "model-00012-of-00030.safetensors": { "bytes": 151036360, "sha256": "3e6b80b7fd596937dbb5cbd044dfe201d89f11869c035afbdbd64b0f207c012c" }, "model-00013-of-00030.safetensors": { "bytes": 151036360, "sha256": "9f0f4f5490946de1a2072c65018b37af1a34b4fc1ce0270078f0efddea274888" }, "model-00014-of-00030.safetensors": { "bytes": 151036360, "sha256": "2f700280b3e3f861aa4b02daf1cfa6992ac912079bf4a56605e483faebc09f97" }, "model-00015-of-00030.safetensors": { "bytes": 151036360, "sha256": "07fee2ec5a5524af7be82be4d01e1082afa286c6c1ac0bcad30c71633a7c5a47" }, "model-00016-of-00030.safetensors": { "bytes": 151036360, "sha256": "7e077d11a377ceb6f9d3c5a9ff8c736292b742672489231e788093466efb015e" }, "model-00017-of-00030.safetensors": { "bytes": 151036360, "sha256": "2c8fe791442985f4845e094e8832ee53c70434fb11b5882c720440b16e5ba980" }, "model-00018-of-00030.safetensors": { "bytes": 151036360, "sha256": "c68fe8b0fc658c7a3dd6fa412c7a663f8ae2accfb0c0c8b3754d84f890e05e0f" }, "model-00019-of-00030.safetensors": { "bytes": 151036360, "sha256": "9daa126c8182566d78d7510b324f797531c8330e9862811c523a9fcef669fe9c" }, "model-00020-of-00030.safetensors": { "bytes": 151036360, "sha256": "eba825e18f72e3d3ff494c6e6843cbc98ddcba4111f646d424d0751f924daa07" }, "model-00021-of-00030.safetensors": { "bytes": 151036360, "sha256": "2abb8005e28426b315871100ae5d6fbfc7420c1538af72b4a0ed60991579abfa" }, "model-00022-of-00030.safetensors": { "bytes": 151036360, "sha256": "6da6c4074d34c98aa021e7d5a167f7d1b9a77e0eaedf55234a721290f6e0469e" }, "model-00023-of-00030.safetensors": { "bytes": 151036360, "sha256": "7622885277d1d2077463c1b5bdf496db7710e55b1f43eb4fe090474e14b410a2" }, "model-00024-of-00030.safetensors": { "bytes": 151036360, "sha256": "d0d85d88f85dcae50735b1ebe788044e31a97fe0f728deecde3b892b90a9039b" }, "model-00025-of-00030.safetensors": { "bytes": 151036360, "sha256": "a6ab021ef46a4dc116e83d3ce4627bb0de3bd7f6b81ae31fe4ecae947b804d26" }, "model-00026-of-00030.safetensors": { "bytes": 151036360, "sha256": "ab1717c050eafba9d6b0944d25149d1d5146c46892e4d9177acf18fd4dbd59a0" }, "model-00027-of-00030.safetensors": { "bytes": 151036360, "sha256": "79ba8e3e28d09ad3d2e430f7494cf9680d2bc71fc9ee5bf4b4e3aa857d51d18f" }, "model-00028-of-00030.safetensors": { "bytes": 151036360, "sha256": "148ec003b8fb3f2f3252712a57c2ed4cdd32322ac5be1f30414f375e7c5997e7" }, "model-00029-of-00030.safetensors": { "bytes": 151036360, "sha256": "b7214c9bcef54934778158ec51d9715972784f03208d480fd78b5060b30a9cd2" }, "model-00030-of-00030.safetensors": { "bytes": 637184, "sha256": "413feaa8a5d1ccf9554576b8715c994944a640ba0e9139c5a2598ab0f9dce3ef" }, "model.safetensors.index.json": { "bytes": 14038, "sha256": "175f2123ff5517c90bc5db37897b087ef8ef8579fa4922bcde4d8507a973d2bf" }, "tokenizer.json": { "bytes": 2041, "sha256": "a63388c8362fc109382b66506b666d798b7e0170828d0047d16790978a14b5f0" }, "tokenizer_config.json": { "bytes": 199, "sha256": "6f9df026786c39bfda75becfa45ea655ccad3ee7bf4c4e186777e11aa930758b" }, "torchwright_schematic_support.npz": { "bytes": 909442, "sha256": "a1f439d473c33c5d6099ad815acc0dd18945e58fc49f06cc76d6cfc692098352" }, "torchwright_schematic_v1.schema.json": { "bytes": 3678, "sha256": "1881c7400c00ee66e6e911c7416ceb573b31129cf24bb80d4055b9e2c647ec7f" } }, "kind": "huggingface_causal_lm_bundle" }, "build": { "compiler_code_sha256": "639e918d8b7011d6f247de3e5696cc4faaa3b154fe081a266f956ddaa9e68e1c", "options": { "activation": "swish", "bias": false, "collapse_piecewise_linear": true, "collapse_univariate": true, "d_head": 64, "d_hidden_capacity": 4096, "d_model": 2048, "n_heads_capacity": 32, "optimize": 2, "trim_heads": true }, "schedule_fingerprint": "eb50446425f1908588859783fa8f0bc7ce59ffc72b35c88bfed766caaa07b508", "schedule_provenance": { "delivery": "fresh", "optimize": 2, "selected_is_optimal": true, "selected_objective": 29, "selected_objective_blocks": [ 29, 0 ], "selected_origin": "solver", "solver_best_bound": 29.0, "solver_is_optimal": true, "solver_objective": 29, "solver_status": "OPTIMAL" }, "torchwright_version": "unknown" }, "format": "torchwright.schematic.v1", "graphs": { "internal_nodes": [ { "id": "i:0", "name": "rope_self_match_const_one", "op": "LiteralValue", "purpose": "compiler_internal", "width": 1 } ], "lowered": { "nodes": [ { "annotation": null, "id": "l:0", "inputs": [ "l:1" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "3422cbe5c1e8b5b1e016e89ff3387bd4290361fdb17782c50e53c187cf575c5d", "shape": [ 34, 35 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e0ebe36501a3a3bd49b9c2eca7f757e673188e21386e2ef140b112922c14d71b", "shape": [ 34, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7bd2d0249f58c2a4d46659f07872f284b01435b9afd6b0e16d9551cfac29acb6", "shape": [ 34, 35 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 34 }, "source_nodes": [ "s:0" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=146.70925247975623))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 146.70925247975623, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 146.70925247975623, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:1", "inputs": [ "l:2", "l:10", "l:513" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:1" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 146.70925247975623, "upper_unbounded": false } }, "width": 35 }, { "annotation": null, "id": "l:2", "inputs": [ "l:3", "l:6", "l:7" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:2" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.005, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:3", "inputs": [ "l:4", "l:5" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:3" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:4", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:4" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:5", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:5" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:6", "inputs": [ "l:7", "l:9" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:6" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:7", "inputs": [ "l:8" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "3eb8480c9ef49139f499acac7ad953fae7d8c289cb60b001375681e6691ca7d1", "shape": [ 1, 17 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "source_nodes": [ "s:7" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:8", "inputs": [], "name": null, "op": "Embedding", "parameters": [ { "dtype": "float32", "name": "table", "sha256": "6bbd3059883cf8b4699c1da2e25ef31211d490f885fcaac1f73475d14b493c32", "shape": [ 18, 17 ] } ], "scheduling_predecessors": [], "semantics": { "embedding_width": 17, "input_name": "embedding_input", "vocab_size": 18 }, "source_nodes": [ "s:8" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:9", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:9" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:10", "inputs": [ "l:11" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d1eb096a138388c9ee051d92d5ac6d8e55f4754d7e7650310fcab089dad79925", "shape": [ 136, 17 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:10" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 146.70925247975623, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:11", "inputs": [ "l:12", "l:424", "l:445", "l:463", "l:473", "l:483", "l:493", "l:503" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:11" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 36.67731311993905, "upper_unbounded": false } }, "width": 136 }, { "annotation": null, "id": "l:12", "inputs": [ "l:13" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:12" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=18.247419462656246))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 18.247419462656246, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 18.247419462656246, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:13", "inputs": [ "l:14", "l:23" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:13" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 18.247419462656246, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:14", "inputs": [ "l:15" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:14" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:15", "inputs": [ "l:16", "l:22" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:16" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:16", "inputs": [ "l:17" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "25870a3818cab31267591ff755d69e74c8435d6e091e4ff7c8ca9945f2f3285c", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:17" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.9956489921875, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:17", "inputs": [ "l:18" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:18" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.3418431447269631, "lower_unbounded": false, "upper": 9.50850981139363, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:18", "inputs": [ "l:19" ], "name": "reciprocal_0_32", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6c50dd4f13b1539d4f43be532766dbdf7fb95eaa3941166aa039680e2e300a9d", "shape": [ 32 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "6b66e1d03190b21246c3ec82a52fd11b0466bbcf48208c4e3f55fb2eaec6cc20", "shape": [ 32, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "483afcc6ea8b039d16cfd49cb47e300e3ec28b944e85daf9d068c4c79a838894", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "60ace6d30c5997f9dc18846fc3aac46f4e0ed2220c14b4c15fcb4f3b0cb7a036", "shape": [ 32, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 32 }, "source_nodes": [ "s:19" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.6581568552730369, hi=10.50850981139363))" } ], "claimed_range": { "lower": 0.6581568552730369, "lower_unbounded": false, "upper": 10.50850981139363, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.6581568552730369, "lower_unbounded": false, "upper": 10.50850981139363, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:19", "inputs": [ "l:20", "l:2", "l:21" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "5138effe2b38a13d44fc24c489380fdb2cb5c5b4b28dacceb3e715e35be86483", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "2e9dafc0fe03a4248692ed626f839acc963a37bac252e72fb9b8a38c20074d31", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:20" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "values in [-0.002175503906249987, 1.0]" } ], "claimed_range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:20", "inputs": [], "name": "mean_where_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:21" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:21", "inputs": [ "l:7" ], "name": "fused_add_const_multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:22" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:22", "inputs": [ "l:17" ], "name": "fused_negate_compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "25870a3818cab31267591ff755d69e74c8435d6e091e4ff7c8ca9945f2f3285c", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bedf28cd77bc6a0aa72157aa10d604f1203eb0409b546f9a088311112009bb5b", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:24" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:23", "inputs": [ "l:24" ], "name": "remove_leading_0s_sum", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d7c028b7eda356619d577d7e3dfef5da2267e2c58f989f7adcb5316c4793dd4c", "shape": [ 204, 17 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:26" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 18.247419462656246, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:24", "inputs": [ "l:25" ], "name": "broadcast_select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2c93ccfa935dc7b48ae36ca9b98d2c3e3e50962957d9f3b464865f5c129d7072", "shape": [ 204, 216 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "08720f852c879614691b2c33f2980888e7cdf29871f0a4e9e60e7446758bfc2e", "shape": [ 204, 204 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7e26b5245da34ef2eb9e8e57159215ac9a011c4de6a4aa42cdbfb51763b7de8f", "shape": [ 204, 216 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 204 }, "source_nodes": [ "s:27" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0149999999999997))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "l:25", "inputs": [ "l:26", "l:405" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:28" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 216 }, { "annotation": null, "id": "l:26", "inputs": [ "l:27" ], "name": "collapse_pl_Add#457_in_range_0_46", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "7580ae8af901cf34a2b10c1f27c0982c8b3075a6771a529f3d120d819901dee9", "shape": [ 46 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "cd78aa3e39d231259d577b69360aab9be0901ee17b98b174e5082aa1fa40e30d", "shape": [ 46, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "e2c1e476e4d39f998a1474e3e82df6cd962dac65d311e65d5a0c29d53b6dee2c", "shape": [ 12 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "7bfd681b2968770c559fa175fc27c4faab6c6218fef01a3adf715374899523f6", "shape": [ 46, 12 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 46 }, "source_nodes": [ "s:29" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0005074638340876, hi=1.0005073449969))" } ], "claimed_range": { "lower": -1.0005074638340876, "lower_unbounded": false, "upper": 1.0005073449969, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0005074638340876, "lower_unbounded": false, "upper": 1.0005073449969, "upper_unbounded": false } }, "width": 12 }, { "annotation": null, "id": "l:27", "inputs": [ "l:28", "l:401" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:31" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.023930542968749968, "lower_unbounded": false, "upper": 11.023930542968749, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:28", "inputs": [ "l:29" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "072e3304b03423a4767d28c5fed09f81d5190ff60a3d078c6c1350eeb8bee28b", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "76cb54bedbda438629b40aa213e8fcdaa3a961e6c305ff2df2b28ab97d143335", "shape": [ 5, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:32" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.010877519531249824, "lower_unbounded": false, "upper": 5.01087751953125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:29", "inputs": [ "l:30", "l:270", "l:292", "l:323", "l:380" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 5 }, { "annotation": null, "id": "l:30", "inputs": [ "l:31" ], "name": "collapse_pl_equals_vector_compare_0_4", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0716ec39c7103bc68885c4c38eab888048f12e4a511d3c97f652324e25e1a490", "shape": [ 4 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a0c210f3fab053052f1b881e15ec97fe9fb4c75e26c09087eb9a2121b689d359", "shape": [ 4, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "a1893dbc217bd565883e63e6a5b175b772bbfd7ebd0b9c0394e0663245cca60b", "shape": [ 4, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 4 }, "source_nodes": [ "s:36" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0005, hi=1.0005))" } ], "claimed_range": { "lower": -1.0005, "lower_unbounded": false, "upper": 1.0005, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0005, "lower_unbounded": false, "upper": 1.0005, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:31", "inputs": [ "l:32" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d02095ef9169de77f0d4bcaefe3494a01a6cb5303d8f16874d410303fff7354a", "shape": [ 1, 17 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "source_nodes": [ "s:41" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:32", "inputs": [ "l:33" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9de7dcac2491cd60a9a20fa1b13cbaa6a2ae2547b251a17e2001b747a444d46d", "shape": [ 51, 17 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:42" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:33", "inputs": [ "l:34", "l:60", "l:69" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:43" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 51 }, { "annotation": null, "id": "l:34", "inputs": [ "l:35" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:44" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:35", "inputs": [ "l:36", "l:59" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:45" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:36", "inputs": [ "l:37", "l:40", "l:46" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:46" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.005, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:37", "inputs": [ "l:38", "l:39" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:47" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:38", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:48" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:39", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:49" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:40", "inputs": [ "l:41", "l:58" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:50" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:41", "inputs": [ "l:42" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:51" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:42", "inputs": [ "l:43", "l:51" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:53" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:43", "inputs": [ "l:44" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "28c4f7df3425ac5087c2632d76786a4afaaf06906eeafb89d8d36ba650e3f24b", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "9fe7252ae4cff6be4d42fdf46fd404e91cf1503a785d6ee2cfc0779c6e4c7a57", "shape": [ 2, 3 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:54" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:44", "inputs": [ "l:45", "l:47", "l:49" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:56" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:45", "inputs": [ "l:46" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "aced851223ade84128237ff20c2acca7ac1102b3d2dfea9f7fcfb3eef3457a01", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:57" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.00217550390625, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:46", "inputs": [ "l:8" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "ef7b1f9e5a2d907c7ff2496a32dc2d4879721c404cc224a31671940ba846c4e2", "shape": [ 1, 17 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "source_nodes": [ "s:58" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:47", "inputs": [ "l:48" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "aced851223ade84128237ff20c2acca7ac1102b3d2dfea9f7fcfb3eef3457a01", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:59" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.00217550390625, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:48", "inputs": [ "l:8" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "82c5b2a5f705dd388e12fac162246ad059cfa99089bdd1df5103928849793813", "shape": [ 1, 17 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "source_nodes": [ "s:60" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:49", "inputs": [ "l:50" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "aced851223ade84128237ff20c2acca7ac1102b3d2dfea9f7fcfb3eef3457a01", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:61" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.00217550390625, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:50", "inputs": [ "l:8" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2e5d5c3fc5f63a23d29988ab3982c8da9a63a8cd2639d8ca27ab5390529327d9", "shape": [ 1, 17 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "source_nodes": [ "s:62" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:51", "inputs": [ "l:52" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "f41491c367f49e7206c45724651464310fcb7d17e3059779d1c95aa901f3fce1", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:63" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:52", "inputs": [ "l:53", "l:56", "l:7" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:64" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:53", "inputs": [ "l:54", "l:55" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:65" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:54", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:66" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:55", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:67" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:56", "inputs": [ "l:7", "l:57" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:68" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:57", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:69" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:58", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:70" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:59", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ], "values": [ 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:71" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:60", "inputs": [ "l:61" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:72" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:61", "inputs": [ "l:62", "l:68" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:73" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:62", "inputs": [ "l:63", "l:66", "l:48" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:74" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.005, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:63", "inputs": [ "l:64", "l:65" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:75" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:64", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:76" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:65", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:77" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:66", "inputs": [ "l:41", "l:67" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:78" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:67", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:79" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:68", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ], "values": [ 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:80" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:69", "inputs": [ "l:70" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:81" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:70", "inputs": [ "l:71", "l:77" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:82" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:71", "inputs": [ "l:72", "l:75", "l:50" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:83" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.005, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:72", "inputs": [ "l:73", "l:74" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:84" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:73", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:85" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:74", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:86" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:75", "inputs": [ "l:41", "l:76" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:87" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:76", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:88" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:77", "inputs": [ "l:78" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "ed95422d29540bfc7f36e115c60ffa7a8494973d7cfe886d9611f90169997399", "shape": [ 200, 17 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "source_nodes": [ "s:89" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:78", "inputs": [ "l:79", "l:179", "l:202" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:90" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 36 }, { "annotation": null, "id": "l:79", "inputs": [ "l:80" ], "name": "collapse_pl_Linear#427_fused_fused_add_const_multiply_const_onehot_lookup_select_0_146", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "070271469f95bb797dc5a791c20918fe37c64a3859e4a51f6a4253c52531283b", "shape": [ 146 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2df038a095237ae3374f93c9de66f4af7913190ca16d9b7b6e34e1db064a3112", "shape": [ 146, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "acf6d30e3b4ddc9fa3d6305ba29ab1edd5021ebaca2c9bd647b74e06e39f4361", "shape": [ 146, 17 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 146 }, "source_nodes": [ "s:91" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0005149277173835658, hi=1.0005149276843444))" } ], "claimed_range": { "lower": -0.0005149277173835658, "lower_unbounded": false, "upper": 1.0005149276843444, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0005149277173835658, "lower_unbounded": false, "upper": 1.0005149276843444, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:80", "inputs": [ "l:81" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f6df0810341a84e6547246fc25bdcbd050ddae1571d30f73e318767b4e484335", "shape": [ 34, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:96" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 36.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:81", "inputs": [ "l:82" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "a6b06fbd738b910030a0847ec7779449c3c076d0975def1ce5d4755431bb9ade", "shape": [ 34 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "bc9b478bb7986ee8a1778fc116c22bb23e7e7352444e62a46c7953e07a2b57bd", "shape": [ 100, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "source_nodes": [ "s:100" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:82", "inputs": [ "l:83", "l:135" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:101" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:83", "inputs": [ "l:84", "l:87", "l:89" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 17, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:102" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:84", "inputs": [ "l:85", "l:86" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:103" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:85", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:104" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:86", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:105" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:87", "inputs": [ "l:7", "l:88" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:106" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:88", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:107" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:89", "inputs": [ "l:90", "l:109", "l:132" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "ff1247d951578e37ef4f20b3e4a593814438971e903430bf33e180aaeed8c538", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "54e4e0a85bc6a3e7d002e3c35b74ff13c2449639e18520e333933b32d1fb004a", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 17, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:108" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:90", "inputs": [ "l:91" ], "name": "collapse_pl_Attn#152_m224_0_9", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "3fdefc9ed00f00dea0599794d1c9a69cee3f0f6849c87508fff2da9d6bbb3ddb", "shape": [ 9 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "b05e3b5ab8b2d409e8b3d5f16ea3c8191e3638bbb3168f525a428a2af82fba6b", "shape": [ 9, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "7ab8f6c26e4f9862c95a18c8e5c50403eeb64d8869fbbf9a7a6397d9a63b7b0e", "shape": [ 4 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "eaa82dd1f86788890259eaf455712f67cccbe5433ad98eac3cad71e322d8d347", "shape": [ 9, 4 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 9 }, "source_nodes": [ "s:109" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0009574656486511231, hi=1.0009574060440063))" } ], "claimed_range": { "lower": -0.0009574656486511231, "lower_unbounded": false, "upper": 1.0009574060440063, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0009574656486511231, "lower_unbounded": false, "upper": 1.0009574060440063, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:91", "inputs": [ "l:92", "l:95", "l:97" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:116" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.3366749232420483, "lower_unbounded": false, "upper": 4.503341589908715, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:92", "inputs": [ "l:93", "l:94" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:117" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:93", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:118" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:94", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:119" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:95", "inputs": [ "l:41", "l:96" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:120" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:96", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:121" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:97", "inputs": [ "l:98" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:122" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.3366749232420483, "lower_unbounded": false, "upper": 4.503341589908715, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:98", "inputs": [ "l:99" ], "name": "reciprocal_0_32", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "86ace85102b9d8a1c0886e9cf2cbb181c89f8ca60093d8ccbe6cbd12a754ad1b", "shape": [ 32 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "5870a33a3d94db2db2aaa92ac9edb125e7803a503405c199042a2f0c6d461a6b", "shape": [ 32, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "41d043a7c0f00b8b39f1365727f09ede61e4b865694b6dc1d4acbbd97029d11b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "a4d06da2357859d57b942afb3157f9957beb9485b57a8ae112cd6e8f5e6b3764", "shape": [ 32, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 32 }, "source_nodes": [ "s:123" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.6633250767579517, hi=5.503341589908715))" } ], "claimed_range": { "lower": 0.6633250767579517, "lower_unbounded": false, "upper": 5.503341589908715, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.6633250767579517, "lower_unbounded": false, "upper": 5.503341589908715, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:99", "inputs": [ "l:100", "l:101", "l:108" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "5138effe2b38a13d44fc24c489380fdb2cb5c5b4b28dacceb3e715e35be86483", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "2e9dafc0fe03a4248692ed626f839acc963a37bac252e72fb9b8a38c20074d31", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:124" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "values in [-0.002175503906249987, 1.0]" } ], "claimed_range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:100", "inputs": [], "name": "mean_where_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:125" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:101", "inputs": [ "l:102", "l:105", "l:106" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:126" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:102", "inputs": [ "l:103", "l:104" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:127" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:103", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:128" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:104", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:129" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:105", "inputs": [ "l:106", "l:107" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:130" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:106", "inputs": [ "l:8" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "e79f394c9a536f65c56b29d1e593771c6e332b653623054cfde6b4c6056348bf", "shape": [ 1, 17 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "source_nodes": [ "s:131" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:107", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:132" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:108", "inputs": [ "l:106" ], "name": "fused_add_const_multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:133" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:109", "inputs": [ "l:110", "l:129" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:138" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004372762851562474, "lower_unbounded": false, "upper": 1.5118727628515622, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:110", "inputs": [ "l:111", "l:128" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:139" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004372762851562474, "lower_unbounded": false, "upper": 1.0093727628515623, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:111", "inputs": [ "l:112" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "acc2e4c62e9b92c67550621c1d3c42498eab8b87200286819aae131530c4dc41", "shape": [ 3, 4 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e5cd6f63a143d03bf475cd19b883bbdecec4efb62d8303f209b0dde6ae0565cd", "shape": [ 3, 3 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "ba88de7e97f50c001a9d0c4e5c690cd5e4d90c7e5a7b64b96b485a3847228885", "shape": [ 3, 4 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 3 }, "source_nodes": [ "s:140" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.004351007812499974, hi=1.0043510078125))" } ], "claimed_range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:112", "inputs": [ "l:113", "l:123" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:141" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:113", "inputs": [ "l:114" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:142" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:114", "inputs": [ "l:115", "l:116" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:144" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.04351015625, "lower_unbounded": false, "upper": 19.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:115", "inputs": [ "l:8" ], "name": "map_to_table", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "2c34ce1df23b838c5abf2a7f6437cca3d3067ed509ff25f11df6b11b582b51eb", "shape": [ 10 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "eee183eff6e72006bb7ace8c5ae3f42029abaa0004524c553789b6cca053a22c", "shape": [ 10, 17 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e7a2f4b7a886aabf181b94c496dabd00bd3f8da32a01f3da03e6d6868564aed6", "shape": [ 10, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 10 }, "source_nodes": [ "s:145" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-21.0, hi=19.0))" } ], "claimed_range": { "lower": -21.0, "lower_unbounded": false, "upper": 19.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.04351015625, "lower_unbounded": false, "upper": 19.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:116", "inputs": [ "l:117" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "f41491c367f49e7206c45724651464310fcb7d17e3059779d1c95aa901f3fce1", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:146" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:117", "inputs": [ "l:118", "l:121", "l:41" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:147" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:118", "inputs": [ "l:119", "l:120" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:148" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:119", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:149" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:120", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:150" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:121", "inputs": [ "l:41", "l:122" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:151" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:122", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:152" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:123", "inputs": [ "l:124" ], "name": "fused_add_const_multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "fbfe4ca61ec299680879894a16ca776f81f23454312659ac8736249aa1e9722b", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "56e475e08e4d688e9ca9d3faa1c854f47fc5b6382bdec3ccc4e64114e57ad6e9", "shape": [ 3, 3 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:153" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:124", "inputs": [ "l:125" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "10f5b23f39f81bf5063c468e6bb37dbb7cd44d7381dbc28b54c11a333e817e82", "shape": [ 12 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "1ad43993d87c26d5c610bb4df0a3f9b1b03186bb9c69b26284c08db4a57dfa2e", "shape": [ 12, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3ee6949226ed15c57a63c066edaaac5d2a2988e2ba82c56d47f3d1b7f8620845", "shape": [ 12, 3 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 12 }, "source_nodes": [ "s:155" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:125", "inputs": [ "l:126", "l:127" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:156" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.3366749232420483, "lower_unbounded": false, "upper": 4.503341589908715, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:126", "inputs": [ "l:97" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:157" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.3366749232420483, "lower_unbounded": false, "upper": 3.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:127", "inputs": [ "l:126" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:158" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.3366749232420483, "lower_unbounded": false, "upper": 4.503341589908715, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:128", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ], "values": [ 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:159" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:129", "inputs": [ "l:130" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb", "shape": [ 4 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "6ad01752d1a3928862bafa2e6bbe6b2df45d614b086e065fdacf50cb7180f6f3", "shape": [ 4, 5 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb", "shape": [ 4 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "c91315e209cc3cd1c12fae1264b1597c61d8ddc4fb51cc0941c96fd1be2076b1", "shape": [ 4, 4 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb", "shape": [ 4 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "6e92b4c838f9ab461fc45d72308d5a5360af4497fff417bae914541608979a02", "shape": [ 4, 5 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 4 }, "source_nodes": [ "s:160" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:130", "inputs": [ "l:106", "l:131" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:161" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 5 }, { "annotation": null, "id": "l:131", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "024712320e6f0d6455efd690279f94b44c2b7ac0f5351f626965347a9d3d4199", "shape": [ 4 ], "values": [ 0.5, 0.5, 0.5, 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:162" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.5, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:132", "inputs": [ "l:133" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "3422cbe5c1e8b5b1e016e89ff3387bd4290361fdb17782c50e53c187cf575c5d", "shape": [ 34, 35 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e0ebe36501a3a3bd49b9c2eca7f757e673188e21386e2ef140b112922c14d71b", "shape": [ 34, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7bd2d0249f58c2a4d46659f07872f284b01435b9afd6b0e16d9551cfac29acb6", "shape": [ 34, 35 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 34 }, "source_nodes": [ "s:163" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:133", "inputs": [ "l:113", "l:8", "l:134" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:164" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 35 }, { "annotation": null, "id": "l:134", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ], "values": [ 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:165" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:135", "inputs": [ "l:136", "l:139", "l:141" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 17, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:166" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:136", "inputs": [ "l:137", "l:138" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:167" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:137", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:168" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:138", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:169" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:139", "inputs": [ "l:7", "l:140" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:170" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:140", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:171" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:141", "inputs": [ "l:142", "l:160", "l:176" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "ff1247d951578e37ef4f20b3e4a593814438971e903430bf33e180aaeed8c538", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "54e4e0a85bc6a3e7d002e3c35b74ff13c2449639e18520e333933b32d1fb004a", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 17, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:172" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:142", "inputs": [ "l:143" ], "name": "collapse_pl_Attn#100_m210_0_9", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "3fdefc9ed00f00dea0599794d1c9a69cee3f0f6849c87508fff2da9d6bbb3ddb", "shape": [ 9 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "b05e3b5ab8b2d409e8b3d5f16ea3c8191e3638bbb3168f525a428a2af82fba6b", "shape": [ 9, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "7ab8f6c26e4f9862c95a18c8e5c50403eeb64d8869fbbf9a7a6397d9a63b7b0e", "shape": [ 4 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "eaa82dd1f86788890259eaf455712f67cccbe5433ad98eac3cad71e322d8d347", "shape": [ 9, 4 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 9 }, "source_nodes": [ "s:173" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0009574656486511231, hi=1.0009574060440063))" } ], "claimed_range": { "lower": -0.0009574656486511231, "lower_unbounded": false, "upper": 1.0009574060440063, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0009574656486511231, "lower_unbounded": false, "upper": 1.0009574060440063, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:143", "inputs": [ "l:144", "l:147", "l:149" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:180" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.3366749232420483, "lower_unbounded": false, "upper": 4.503341589908715, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:144", "inputs": [ "l:145", "l:146" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:181" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:145", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:182" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:146", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:183" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:147", "inputs": [ "l:7", "l:148" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:184" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:148", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:185" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:149", "inputs": [ "l:150" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:186" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.3366749232420483, "lower_unbounded": false, "upper": 4.503341589908715, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:150", "inputs": [ "l:151" ], "name": "reciprocal_0_32", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "86ace85102b9d8a1c0886e9cf2cbb181c89f8ca60093d8ccbe6cbd12a754ad1b", "shape": [ 32 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "5870a33a3d94db2db2aaa92ac9edb125e7803a503405c199042a2f0c6d461a6b", "shape": [ 32, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "41d043a7c0f00b8b39f1365727f09ede61e4b865694b6dc1d4acbbd97029d11b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "a4d06da2357859d57b942afb3157f9957beb9485b57a8ae112cd6e8f5e6b3764", "shape": [ 32, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 32 }, "source_nodes": [ "s:187" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.6633250767579517, hi=5.503341589908715))" } ], "claimed_range": { "lower": 0.6633250767579517, "lower_unbounded": false, "upper": 5.503341589908715, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.6633250767579517, "lower_unbounded": false, "upper": 5.503341589908715, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:151", "inputs": [ "l:152", "l:153", "l:159" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "5138effe2b38a13d44fc24c489380fdb2cb5c5b4b28dacceb3e715e35be86483", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "2e9dafc0fe03a4248692ed626f839acc963a37bac252e72fb9b8a38c20074d31", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:188" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "values in [-0.002175503906249987, 1.0021755039062499]" } ], "claimed_range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:152", "inputs": [], "name": "mean_where_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:189" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:153", "inputs": [ "l:154", "l:157", "l:41" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:190" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:154", "inputs": [ "l:155", "l:156" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:191" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:155", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:192" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:156", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:193" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:157", "inputs": [ "l:41", "l:158" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:194" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:158", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:195" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:159", "inputs": [ "l:41" ], "name": "fused_add_const_multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:196" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:160", "inputs": [ "l:161", "l:173" ], "name": null, "op": "Add", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:201" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004372762851562474, "lower_unbounded": false, "upper": 1.5118727628515622, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:161", "inputs": [ "l:162", "l:172" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:202" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004372762851562474, "lower_unbounded": false, "upper": 1.0093727628515623, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:162", "inputs": [ "l:163" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "acc2e4c62e9b92c67550621c1d3c42498eab8b87200286819aae131530c4dc41", "shape": [ 3, 4 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e5cd6f63a143d03bf475cd19b883bbdecec4efb62d8303f209b0dde6ae0565cd", "shape": [ 3, 3 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "ba88de7e97f50c001a9d0c4e5c690cd5e4d90c7e5a7b64b96b485a3847228885", "shape": [ 3, 4 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 3 }, "source_nodes": [ "s:203" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.004351007812499974, hi=1.0043510078125))" } ], "claimed_range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:163", "inputs": [ "l:164", "l:167" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:204" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:164", "inputs": [ "l:165" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "f22239942c4ac043eb6887cd996054943f2d24a5e56217c21186a366ca97c23d", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "9fe7252ae4cff6be4d42fdf46fd404e91cf1503a785d6ee2cfc0779c6e4c7a57", "shape": [ 2, 3 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:205" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:165", "inputs": [ "l:115", "l:117", "l:166" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:207" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.04351015625, "lower_unbounded": false, "upper": 19.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:166", "inputs": [ "l:52" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "f41491c367f49e7206c45724651464310fcb7d17e3059779d1c95aa901f3fce1", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:208" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:167", "inputs": [ "l:168" ], "name": "fused_add_const_multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "fbfe4ca61ec299680879894a16ca776f81f23454312659ac8736249aa1e9722b", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "56e475e08e4d688e9ca9d3faa1c854f47fc5b6382bdec3ccc4e64114e57ad6e9", "shape": [ 3, 3 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:209" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:168", "inputs": [ "l:169" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "10f5b23f39f81bf5063c468e6bb37dbb7cd44d7381dbc28b54c11a333e817e82", "shape": [ 12 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "1ad43993d87c26d5c610bb4df0a3f9b1b03186bb9c69b26284c08db4a57dfa2e", "shape": [ 12, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3ee6949226ed15c57a63c066edaaac5d2a2988e2ba82c56d47f3d1b7f8620845", "shape": [ 12, 3 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 12 }, "source_nodes": [ "s:211" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:169", "inputs": [ "l:170", "l:171" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:212" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.3366749232420483, "lower_unbounded": false, "upper": 4.503341589908715, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:170", "inputs": [ "l:149" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:213" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.3366749232420483, "lower_unbounded": false, "upper": 3.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:171", "inputs": [ "l:170" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:214" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.3366749232420483, "lower_unbounded": false, "upper": 4.503341589908715, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:172", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ], "values": [ 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:215" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:173", "inputs": [ "l:174" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb", "shape": [ 4 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "6ad01752d1a3928862bafa2e6bbe6b2df45d614b086e065fdacf50cb7180f6f3", "shape": [ 4, 5 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb", "shape": [ 4 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "c91315e209cc3cd1c12fae1264b1597c61d8ddc4fb51cc0941c96fd1be2076b1", "shape": [ 4, 4 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb", "shape": [ 4 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "6e92b4c838f9ab461fc45d72308d5a5360af4497fff417bae914541608979a02", "shape": [ 4, 5 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 4 }, "source_nodes": [ "s:216" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:174", "inputs": [ "l:41", "l:175" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:217" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 5 }, { "annotation": null, "id": "l:175", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "024712320e6f0d6455efd690279f94b44c2b7ac0f5351f626965347a9d3d4199", "shape": [ 4 ], "values": [ 0.5, 0.5, 0.5, 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:218" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.5, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:176", "inputs": [ "l:177" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "3422cbe5c1e8b5b1e016e89ff3387bd4290361fdb17782c50e53c187cf575c5d", "shape": [ 34, 35 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e0ebe36501a3a3bd49b9c2eca7f757e673188e21386e2ef140b112922c14d71b", "shape": [ 34, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7bd2d0249f58c2a4d46659f07872f284b01435b9afd6b0e16d9551cfac29acb6", "shape": [ 34, 35 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 34 }, "source_nodes": [ "s:219" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:177", "inputs": [ "l:164", "l:8", "l:178" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:220" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 35 }, { "annotation": null, "id": "l:178", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ], "values": [ 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:221" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:179", "inputs": [ "l:180" ], "name": "collapse_pl_Linear#398_onehot_lookup_select_0_42", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "092bc571fab616bb72d167976c722aefb0c4fe74c6a3add7c0ba09b0223cbd63", "shape": [ 42 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a0e42f96c3dfa4b8fac7a6d58ec25a834198c468a550c40ab2bd57e19b18c28f", "shape": [ 42, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "d1c68a52998223d2a426c5e321afcb2c216403b8b5a31f4777796a31cf695e9e", "shape": [ 42, 17 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 42 }, "source_nodes": [ "s:228" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0005303644553223688, hi=1.0005302171140107))" } ], "claimed_range": { "lower": -0.0005303644553223688, "lower_unbounded": false, "upper": 1.0005302171140107, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0005303644553223688, "lower_unbounded": false, "upper": 1.0005302171140107, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:180", "inputs": [ "l:181" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "ad620f232019cd92e2567785c4f67cd56aded25af9a6428192d000bd529da978", "shape": [ 102, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:233" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 117.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:181", "inputs": [ "l:81", "l:182", "l:192" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 102 }, { "annotation": null, "id": "l:182", "inputs": [ "l:183" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "a6b06fbd738b910030a0847ec7779449c3c076d0975def1ce5d4755431bb9ade", "shape": [ 34 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "bc9b478bb7986ee8a1778fc116c22bb23e7e7352444e62a46c7953e07a2b57bd", "shape": [ 100, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "source_nodes": [ "s:239" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:183", "inputs": [ "l:83", "l:184" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:240" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:184", "inputs": [ "l:185", "l:188", "l:190" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 17, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:241" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:185", "inputs": [ "l:186", "l:187" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:242" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:186", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:243" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:187", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:244" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:188", "inputs": [ "l:7", "l:189" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:245" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:189", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:246" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:190", "inputs": [ "l:191", "l:160", "l:176" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "ff1247d951578e37ef4f20b3e4a593814438971e903430bf33e180aaeed8c538", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "54e4e0a85bc6a3e7d002e3c35b74ff13c2449639e18520e333933b32d1fb004a", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 17, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:247" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:191", "inputs": [ "l:143" ], "name": "collapse_pl_Attn#100_m178_0_13", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "01afa9575aaff88a9f3a51a4578114a24a66192bf8355f33ac56fee5616b18ca", "shape": [ 13 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "fb694bb84d282aed16c503a31849c67f82e5982a0fcf9ac863c4f3998a513076", "shape": [ 13, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "7ab8f6c26e4f9862c95a18c8e5c50403eeb64d8869fbbf9a7a6397d9a63b7b0e", "shape": [ 4 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "1f400d61c0e178ba2a3d7b83d01a8a931c0d44dc581ca40c6549b6acb82fb385", "shape": [ 13, 4 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 13 }, "source_nodes": [ "s:248" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0009574656486511231, hi=1.0009574060440063))" } ], "claimed_range": { "lower": -0.0009574656486511231, "lower_unbounded": false, "upper": 1.0009574060440063, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0009574656486511231, "lower_unbounded": false, "upper": 1.0009574060440063, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:192", "inputs": [ "l:193" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "a6b06fbd738b910030a0847ec7779449c3c076d0975def1ce5d4755431bb9ade", "shape": [ 34 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "bc9b478bb7986ee8a1778fc116c22bb23e7e7352444e62a46c7953e07a2b57bd", "shape": [ 100, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "source_nodes": [ "s:259" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:193", "inputs": [ "l:194", "l:135" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:260" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:194", "inputs": [ "l:195", "l:198", "l:200" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 17, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:261" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:195", "inputs": [ "l:196", "l:197" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:262" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:196", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:263" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:197", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:264" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:198", "inputs": [ "l:7", "l:199" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:265" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:199", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:266" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:200", "inputs": [ "l:201", "l:109", "l:132" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "ff1247d951578e37ef4f20b3e4a593814438971e903430bf33e180aaeed8c538", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "54e4e0a85bc6a3e7d002e3c35b74ff13c2449639e18520e333933b32d1fb004a", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 17, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:267" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:201", "inputs": [ "l:91" ], "name": "collapse_pl_Attn#152_m192_0_13", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "01afa9575aaff88a9f3a51a4578114a24a66192bf8355f33ac56fee5616b18ca", "shape": [ 13 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "fb694bb84d282aed16c503a31849c67f82e5982a0fcf9ac863c4f3998a513076", "shape": [ 13, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "7ab8f6c26e4f9862c95a18c8e5c50403eeb64d8869fbbf9a7a6397d9a63b7b0e", "shape": [ 4 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "1f400d61c0e178ba2a3d7b83d01a8a931c0d44dc581ca40c6549b6acb82fb385", "shape": [ 13, 4 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 13 }, "source_nodes": [ "s:268" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0009574656486511231, hi=1.0009574060440063))" } ], "claimed_range": { "lower": -0.0009574656486511231, "lower_unbounded": false, "upper": 1.0009574060440063, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0009574656486511231, "lower_unbounded": false, "upper": 1.0009574060440063, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:202", "inputs": [ "l:203" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "81e5bd2ad736b785c26f8ee5ad9ebc0d50450a71dfbb241759a0d378dfca6685", "shape": [ 3, 2 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:280" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:203", "inputs": [ "l:204" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "515d1614aef9d13e7b1592189bacbc4129f9986f5af077d91435419ef5889d28", "shape": [ 9 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "8052157cb96bc0305ae47a1a67c53dc01499b9f7c175846e4687adf3addaf1d9", "shape": [ 9, 6 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "480376c6bf738a0227f2bbf2b3506b7cde209152c0ba9a9077e5527169eb292e", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "30e1bbfda7f7c8c9053a5476e4b927c8d24e47e1e4f277056cf7b9b29f266de8", "shape": [ 9, 3 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 9 }, "source_nodes": [ "s:281" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:204", "inputs": [ "l:205", "l:232" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:282" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "l:205", "inputs": [ "l:206" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "480376c6bf738a0227f2bbf2b3506b7cde209152c0ba9a9077e5527169eb292e", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3f34e6886f39376bb37c08c97f7c480d64bc1e54adb1b9083da6ff148def52c6", "shape": [ 100, 3 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "source_nodes": [ "s:283" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:206", "inputs": [ "l:207", "l:231" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:284" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:207", "inputs": [ "l:208" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f29de6d677eb31146fcabd1fd447b4654829f0b547c221f0516d422ce631c077", "shape": [ 100, 17 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:285" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:208", "inputs": [ "l:209" ], "name": "fused_add_const_multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "d9435470614b5f2dad783c3468b5274628aa18c6df3a737b46d6c6e5a050100d", "shape": [ 100 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c79c7241ec19383f6be1709969fd023c0c6afaedf0eb11cb5b8fd6e8b99856d5", "shape": [ 100, 100 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:286" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 100 }, { "annotation": null, "id": "l:209", "inputs": [ "l:210" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "5043c9e1d4cb1fea053e49e1d7a8e09157add36a5a1f3d8ee90efd200a572900", "shape": [ 400 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2872925994dc7f63708f8a149b44e2bd8c152caca1b07d0d22d1db16fdfa7b9e", "shape": [ 400, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "79fd15545a108bf1260d68e92162b968655b2a63fc31cc35e91d084b4cdb7c95", "shape": [ 100 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "322932091bb64b559083df41f6380723142ffa85606836596f3381bffb456144", "shape": [ 400, 100 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 400 }, "source_nodes": [ "s:288" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 100 }, { "annotation": null, "id": "l:210", "inputs": [ "l:211", "l:230" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:289" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 46.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:211", "inputs": [ "l:212" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "5a6f7cc0189691c1ee67233d381e002b0dacdf9a905f1818a63fe2e5d361f2a9", "shape": [ 34, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:290" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 45.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:212", "inputs": [ "l:213" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "a6b06fbd738b910030a0847ec7779449c3c076d0975def1ce5d4755431bb9ade", "shape": [ 34 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "bc9b478bb7986ee8a1778fc116c22bb23e7e7352444e62a46c7953e07a2b57bd", "shape": [ 100, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "source_nodes": [ "s:294" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:213", "inputs": [ "l:214", "l:222" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:295" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:214", "inputs": [ "l:215", "l:218", "l:220" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 17, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:296" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:215", "inputs": [ "l:216", "l:217" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:297" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:216", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:298" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:217", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:299" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:218", "inputs": [ "l:7", "l:219" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:300" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:219", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:301" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:220", "inputs": [ "l:221", "l:109", "l:132" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "ff1247d951578e37ef4f20b3e4a593814438971e903430bf33e180aaeed8c538", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "54e4e0a85bc6a3e7d002e3c35b74ff13c2449639e18520e333933b32d1fb004a", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 17, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:302" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:221", "inputs": [ "l:91" ], "name": "collapse_pl_Attn#152_m160_0_17", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "4a02f2d90e882bbcc9baf4d5ee2c3ee11e4452fded3c1c025541ed7749a28172", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "ec23f63c1abc506eff7bd523c8de0d39ff8d1026fc605e895a14c263f015a00b", "shape": [ 17, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "7ab8f6c26e4f9862c95a18c8e5c50403eeb64d8869fbbf9a7a6397d9a63b7b0e", "shape": [ 4 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50829d67c2320f71da81cbc51e04f11877e2884b2ae9b5e562ad1d5f6f7edc93", "shape": [ 17, 4 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 17 }, "source_nodes": [ "s:303" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0009574656486511231, hi=1.0009575252532958))" } ], "claimed_range": { "lower": -0.0009574656486511231, "lower_unbounded": false, "upper": 1.0009575252532958, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0009574656486511231, "lower_unbounded": false, "upper": 1.0009575252532958, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:222", "inputs": [ "l:223", "l:226", "l:228" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 17, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:312" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:223", "inputs": [ "l:224", "l:225" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:313" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:224", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:314" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:225", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:315" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:226", "inputs": [ "l:7", "l:227" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:316" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:227", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:317" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:228", "inputs": [ "l:229", "l:160", "l:176" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "ff1247d951578e37ef4f20b3e4a593814438971e903430bf33e180aaeed8c538", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "54e4e0a85bc6a3e7d002e3c35b74ff13c2449639e18520e333933b32d1fb004a", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] } ], "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 17, "rope_base": 500000.0, "rope_d_rot": 64 }, "source_nodes": [ "s:318" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:229", "inputs": [ "l:143" ], "name": "collapse_pl_Attn#100_m108_0_17", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "4a02f2d90e882bbcc9baf4d5ee2c3ee11e4452fded3c1c025541ed7749a28172", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "ec23f63c1abc506eff7bd523c8de0d39ff8d1026fc605e895a14c263f015a00b", "shape": [ 17, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "7ab8f6c26e4f9862c95a18c8e5c50403eeb64d8869fbbf9a7a6397d9a63b7b0e", "shape": [ 4 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50829d67c2320f71da81cbc51e04f11877e2884b2ae9b5e562ad1d5f6f7edc93", "shape": [ 17, 4 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 17 }, "source_nodes": [ "s:319" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0009574656486511231, hi=1.0009575252532958))" } ], "claimed_range": { "lower": -0.0009574656486511231, "lower_unbounded": false, "upper": 1.0009575252532958, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0009574656486511231, "lower_unbounded": false, "upper": 1.0009575252532958, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:230", "inputs": [ "l:211" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:332" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 46.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:231", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ], "values": [ 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:223" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:232", "inputs": [ "l:233" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "515d1614aef9d13e7b1592189bacbc4129f9986f5af077d91435419ef5889d28", "shape": [ 9 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "8052157cb96bc0305ae47a1a67c53dc01499b9f7c175846e4687adf3addaf1d9", "shape": [ 9, 6 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "480376c6bf738a0227f2bbf2b3506b7cde209152c0ba9a9077e5527169eb292e", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "30e1bbfda7f7c8c9053a5476e4b927c8d24e47e1e4f277056cf7b9b29f266de8", "shape": [ 9, 3 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 9 }, "source_nodes": [ "s:333" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:233", "inputs": [ "l:234", "l:258" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:334" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "l:234", "inputs": [ "l:235" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "515d1614aef9d13e7b1592189bacbc4129f9986f5af077d91435419ef5889d28", "shape": [ 9 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "8052157cb96bc0305ae47a1a67c53dc01499b9f7c175846e4687adf3addaf1d9", "shape": [ 9, 6 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "480376c6bf738a0227f2bbf2b3506b7cde209152c0ba9a9077e5527169eb292e", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "30e1bbfda7f7c8c9053a5476e4b927c8d24e47e1e4f277056cf7b9b29f266de8", "shape": [ 9, 3 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 9 }, "source_nodes": [ "s:335" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:235", "inputs": [ "l:236", "l:246" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:336" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "l:236", "inputs": [ "l:237" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "480376c6bf738a0227f2bbf2b3506b7cde209152c0ba9a9077e5527169eb292e", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3f34e6886f39376bb37c08c97f7c480d64bc1e54adb1b9083da6ff148def52c6", "shape": [ 100, 3 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "source_nodes": [ "s:337" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:237", "inputs": [ "l:238", "l:245" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:338" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:238", "inputs": [ "l:239" ], "name": "collapse_pl_Linear#261_onehot_lookup_select_0_402", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "db86089b10037f6cb57b0bcc52041f0b42451786e13b12d2c69f2526f8a53bea", "shape": [ 402 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "9dfe67554a068f5d27fdf161f3c48977a080e3d61043e709e1e570cd2e0da8ab", "shape": [ 402, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "c6aa9d5bf30bbb2d82efba8b1e03de1e6f99623383f8e43251145e045d9e46f4", "shape": [ 402, 17 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 402 }, "source_nodes": [ "s:339" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.00053037597312258, hi=1.0005290740821196))" } ], "claimed_range": { "lower": -0.00053037597312258, "lower_unbounded": false, "upper": 1.0005290740821196, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.00053037597312258, "lower_unbounded": false, "upper": 1.0005290740821196, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:239", "inputs": [ "l:240" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "053e5c9095d09737c1e9badcbe230e747bf96c9cd70464f6a322c33e1cf56ab9", "shape": [ 102, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:344" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 126.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:240", "inputs": [ "l:241", "l:243", "l:212" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 102 }, { "annotation": null, "id": "l:241", "inputs": [ "l:242" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "a6b06fbd738b910030a0847ec7779449c3c076d0975def1ce5d4755431bb9ade", "shape": [ 34 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "bc9b478bb7986ee8a1778fc116c22bb23e7e7352444e62a46c7953e07a2b57bd", "shape": [ 100, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "source_nodes": [ "s:348" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:242", "inputs": [ "l:194", "l:222" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:349" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:243", "inputs": [ "l:244" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "a6b06fbd738b910030a0847ec7779449c3c076d0975def1ce5d4755431bb9ade", "shape": [ 34 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "bc9b478bb7986ee8a1778fc116c22bb23e7e7352444e62a46c7953e07a2b57bd", "shape": [ 100, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "source_nodes": [ "s:352" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:244", "inputs": [ "l:214", "l:184" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:353" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:245", "inputs": [ "l:208" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "6f8067031a2f2304a92ac0120dc3bace57a8632405b8843002c93959dfbaab6f", "shape": [ 100, 17 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:359" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:246", "inputs": [ "l:247" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "480376c6bf738a0227f2bbf2b3506b7cde209152c0ba9a9077e5527169eb292e", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3f34e6886f39376bb37c08c97f7c480d64bc1e54adb1b9083da6ff148def52c6", "shape": [ 100, 3 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "source_nodes": [ "s:360" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:247", "inputs": [ "l:248", "l:257" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:361" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:248", "inputs": [ "l:249" ], "name": "collapse_pl_Linear#309_onehot_lookup_select_0_402", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "958b5cddad3cba604512e1553e44445969b624111c29f204ccea641fe590c5f3", "shape": [ 402 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "9dfe67554a068f5d27fdf161f3c48977a080e3d61043e709e1e570cd2e0da8ab", "shape": [ 402, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4d1b8d7230ab658bccbde4cf512f94b8ca53137ac98662b9780c76fff558303e", "shape": [ 402, 17 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 402 }, "source_nodes": [ "s:362" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0005303928480330093, hi=1.0005290740821196))" } ], "claimed_range": { "lower": -0.0005303928480330093, "lower_unbounded": false, "upper": 1.0005290740821196, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0005303928480330093, "lower_unbounded": false, "upper": 1.0005290740821196, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:249", "inputs": [ "l:250" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "8ee2dad4a0224657f33d3734f6bdce2dc7f53fe32caa5596ef09391ab12efa44", "shape": [ 170, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:367" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 207.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:250", "inputs": [ "l:251", "l:253", "l:241", "l:255", "l:243" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 170 }, { "annotation": null, "id": "l:251", "inputs": [ "l:252" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "a6b06fbd738b910030a0847ec7779449c3c076d0975def1ce5d4755431bb9ade", "shape": [ 34 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "bc9b478bb7986ee8a1778fc116c22bb23e7e7352444e62a46c7953e07a2b57bd", "shape": [ 100, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "source_nodes": [ "s:371" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:252", "inputs": [ "l:83", "l:222" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:372" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:253", "inputs": [ "l:254" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "a6b06fbd738b910030a0847ec7779449c3c076d0975def1ce5d4755431bb9ade", "shape": [ 34 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "bc9b478bb7986ee8a1778fc116c22bb23e7e7352444e62a46c7953e07a2b57bd", "shape": [ 100, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "source_nodes": [ "s:375" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:254", "inputs": [ "l:194", "l:184" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:376" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:255", "inputs": [ "l:256" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "a6b06fbd738b910030a0847ec7779449c3c076d0975def1ce5d4755431bb9ade", "shape": [ 34 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "bc9b478bb7986ee8a1778fc116c22bb23e7e7352444e62a46c7953e07a2b57bd", "shape": [ 100, 34 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "source_nodes": [ "s:381" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:256", "inputs": [ "l:214", "l:135" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:382" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:257", "inputs": [ "l:239" ], "name": "collapse_pl_Linear#261_onehot_lookup_select_0_42", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "a9a2ef8a000d0a8a52c3f193f9080b973fab63982147d8bf5dc43a140545a501", "shape": [ 42 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a0e42f96c3dfa4b8fac7a6d58ec25a834198c468a550c40ab2bd57e19b18c28f", "shape": [ 42, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "68306fa25b2ea36e61ea73abff2bcf39a32128e8f93553bbc7a6eb8650572efe", "shape": [ 42, 17 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 42 }, "source_nodes": [ "s:386" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.00053037597312258, hi=1.0005302171140107))" } ], "claimed_range": { "lower": -0.00053037597312258, "lower_unbounded": false, "upper": 1.0005302171140107, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.00053037597312258, "lower_unbounded": false, "upper": 1.0005302171140107, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:258", "inputs": [ "l:259" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "515d1614aef9d13e7b1592189bacbc4129f9986f5af077d91435419ef5889d28", "shape": [ 9 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "8052157cb96bc0305ae47a1a67c53dc01499b9f7c175846e4687adf3addaf1d9", "shape": [ 9, 6 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "480376c6bf738a0227f2bbf2b3506b7cde209152c0ba9a9077e5527169eb292e", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "30e1bbfda7f7c8c9053a5476e4b927c8d24e47e1e4f277056cf7b9b29f266de8", "shape": [ 9, 3 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 9 }, "source_nodes": [ "s:387" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:259", "inputs": [ "l:260", "l:266" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:388" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "l:260", "inputs": [ "l:261" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "480376c6bf738a0227f2bbf2b3506b7cde209152c0ba9a9077e5527169eb292e", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3f34e6886f39376bb37c08c97f7c480d64bc1e54adb1b9083da6ff148def52c6", "shape": [ 100, 3 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "source_nodes": [ "s:389" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:261", "inputs": [ "l:262", "l:265" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:390" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:262", "inputs": [ "l:263" ], "name": "collapse_pl_Linear#358_onehot_lookup_select_0_402", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "d76a507985ece1a0d4c8052d314b8493cce8b08db408945896fec3c50fdb915f", "shape": [ 402 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "9dfe67554a068f5d27fdf161f3c48977a080e3d61043e709e1e570cd2e0da8ab", "shape": [ 402, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "bd98e81a29607113155885c1bb7e2ea75ea036065fc82b58729c0e1731b3fdbf", "shape": [ 402, 17 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 402 }, "source_nodes": [ "s:391" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0005303923435941686, hi=1.0005290740821196))" } ], "claimed_range": { "lower": -0.0005303923435941686, "lower_unbounded": false, "upper": 1.0005290740821196, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0005303923435941686, "lower_unbounded": false, "upper": 1.0005290740821196, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:263", "inputs": [ "l:264" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "38a6d8d53a9b52b012a34877296a2e65dbde835ab6eb26d8c12264884a965304", "shape": [ 170, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:396" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 198.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:264", "inputs": [ "l:182", "l:251", "l:192", "l:253", "l:255" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 170 }, { "annotation": null, "id": "l:265", "inputs": [ "l:249" ], "name": "collapse_pl_Linear#309_onehot_lookup_select_0_42", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "25917657e3d394eed18b314a9e7a720b24775bcdc20b8303694b26e087f74f35", "shape": [ 42 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a0e42f96c3dfa4b8fac7a6d58ec25a834198c468a550c40ab2bd57e19b18c28f", "shape": [ 42, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4de105540eb37d430015dd3acd0f1e1b91f5355022a10f142c97a81ead0d92f6", "shape": [ 42, 17 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 42 }, "source_nodes": [ "s:409" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0005303928480330093, hi=1.0005302171140107))" } ], "claimed_range": { "lower": -0.0005303928480330093, "lower_unbounded": false, "upper": 1.0005302171140107, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0005303928480330093, "lower_unbounded": false, "upper": 1.0005302171140107, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:266", "inputs": [ "l:267" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "480376c6bf738a0227f2bbf2b3506b7cde209152c0ba9a9077e5527169eb292e", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3f34e6886f39376bb37c08c97f7c480d64bc1e54adb1b9083da6ff148def52c6", "shape": [ 100, 3 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "source_nodes": [ "s:410" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:267", "inputs": [ "l:268", "l:269" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:411" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:268", "inputs": [ "l:180" ], "name": "collapse_pl_Linear#398_onehot_lookup_select_0_402", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "13bfa9d2ef8c7a7059effb2e37cff3b5a0cb160cfdbdf89656695da838d1faff", "shape": [ 402 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "9dfe67554a068f5d27fdf161f3c48977a080e3d61043e709e1e570cd2e0da8ab", "shape": [ 402, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "15f6a498326dfc60e2b2f267cf5bf85076a02bec3895e93f60b2800309a5c49f", "shape": [ 402, 17 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 402 }, "source_nodes": [ "s:412" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0005303644553223688, hi=1.0005290740821196))" } ], "claimed_range": { "lower": -0.0005303644553223688, "lower_unbounded": false, "upper": 1.0005290740821196, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0005303644553223688, "lower_unbounded": false, "upper": 1.0005290740821196, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:269", "inputs": [ "l:263" ], "name": "collapse_pl_Linear#358_onehot_lookup_select_0_42", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "7ff5c8f9aeb6616340842a0b164b98a3f6d28b34bd91b4da8fdd2eda69877c9c", "shape": [ 42 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a0e42f96c3dfa4b8fac7a6d58ec25a834198c468a550c40ab2bd57e19b18c28f", "shape": [ 42, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "d8187cee4b26edb6d927ff9900c1939b2817fa354667935cb0873f154ce01b9b", "shape": [ 42, 17 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 42 }, "source_nodes": [ "s:413" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.0005303923435941686, hi=1.0005302171140107))" } ], "claimed_range": { "lower": -0.0005303923435941686, "lower_unbounded": false, "upper": 1.0005302171140107, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.0005303923435941686, "lower_unbounded": false, "upper": 1.0005302171140107, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:270", "inputs": [ "l:271" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "eb9f1ff3e4fa10d12a6ae64cce02175007e98fff40f70e2ba8fdbf8f5a123795", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:416" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:271", "inputs": [ "l:272", "l:273" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:418" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:272", "inputs": [ "l:31" ], "name": "fused_add_const_multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:39" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:273", "inputs": [ "l:274" ], "name": "fused_add_const_multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:419" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:274", "inputs": [ "l:275" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d02095ef9169de77f0d4bcaefe3494a01a6cb5303d8f16874d410303fff7354a", "shape": [ 1, 17 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "source_nodes": [ "s:421" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:275", "inputs": [ "l:276" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9de7dcac2491cd60a9a20fa1b13cbaa6a2ae2547b251a17e2001b747a444d46d", "shape": [ 51, 17 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:422" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:276", "inputs": [ "l:277", "l:279", "l:281" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:423" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 51 }, { "annotation": null, "id": "l:277", "inputs": [ "l:278" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:424" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:278", "inputs": [ "l:36", "l:59" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:425" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:279", "inputs": [ "l:280" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:426" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:280", "inputs": [ "l:62", "l:68" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:427" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:281", "inputs": [ "l:282" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:428" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:282", "inputs": [ "l:71", "l:283" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:429" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:283", "inputs": [ "l:284" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "ed95422d29540bfc7f36e115c60ffa7a8494973d7cfe886d9611f90169997399", "shape": [ 200, 17 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "source_nodes": [ "s:430" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:284", "inputs": [ "l:268", "l:269", "l:285" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:431" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 36 }, { "annotation": null, "id": "l:285", "inputs": [ "l:286" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "81e5bd2ad736b785c26f8ee5ad9ebc0d50450a71dfbb241759a0d378dfca6685", "shape": [ 3, 2 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:432" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:286", "inputs": [ "l:287" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "515d1614aef9d13e7b1592189bacbc4129f9986f5af077d91435419ef5889d28", "shape": [ 9 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "8052157cb96bc0305ae47a1a67c53dc01499b9f7c175846e4687adf3addaf1d9", "shape": [ 9, 6 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "480376c6bf738a0227f2bbf2b3506b7cde209152c0ba9a9077e5527169eb292e", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "30e1bbfda7f7c8c9053a5476e4b927c8d24e47e1e4f277056cf7b9b29f266de8", "shape": [ 9, 3 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 9 }, "source_nodes": [ "s:433" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:287", "inputs": [ "l:288", "l:290" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:434" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "l:288", "inputs": [ "l:289" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "515d1614aef9d13e7b1592189bacbc4129f9986f5af077d91435419ef5889d28", "shape": [ 9 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "8052157cb96bc0305ae47a1a67c53dc01499b9f7c175846e4687adf3addaf1d9", "shape": [ 9, 6 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "480376c6bf738a0227f2bbf2b3506b7cde209152c0ba9a9077e5527169eb292e", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "30e1bbfda7f7c8c9053a5476e4b927c8d24e47e1e4f277056cf7b9b29f266de8", "shape": [ 9, 3 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 9 }, "source_nodes": [ "s:435" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:289", "inputs": [ "l:205", "l:236" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:436" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "l:290", "inputs": [ "l:291" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "515d1614aef9d13e7b1592189bacbc4129f9986f5af077d91435419ef5889d28", "shape": [ 9 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "8052157cb96bc0305ae47a1a67c53dc01499b9f7c175846e4687adf3addaf1d9", "shape": [ 9, 6 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "480376c6bf738a0227f2bbf2b3506b7cde209152c0ba9a9077e5527169eb292e", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "30e1bbfda7f7c8c9053a5476e4b927c8d24e47e1e4f277056cf7b9b29f266de8", "shape": [ 9, 3 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 9 }, "source_nodes": [ "s:437" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:291", "inputs": [ "l:246", "l:260" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:438" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "l:292", "inputs": [ "l:293" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1cc95150dbcdc695d9fe82393ccc24788e0bbdb47c092c4100228da7b850f0b9", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "9fe7252ae4cff6be4d42fdf46fd404e91cf1503a785d6ee2cfc0779c6e4c7a57", "shape": [ 2, 3 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:441" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:293", "inputs": [ "l:272", "l:273", "l:294" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:443" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:294", "inputs": [ "l:295" ], "name": "fused_add_const_multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:444" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:295", "inputs": [ "l:296" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d02095ef9169de77f0d4bcaefe3494a01a6cb5303d8f16874d410303fff7354a", "shape": [ 1, 17 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "source_nodes": [ "s:446" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:296", "inputs": [ "l:297" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9de7dcac2491cd60a9a20fa1b13cbaa6a2ae2547b251a17e2001b747a444d46d", "shape": [ 51, 17 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:447" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:297", "inputs": [ "l:298", "l:314", "l:316" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:448" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 51 }, { "annotation": null, "id": "l:298", "inputs": [ "l:299" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:449" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:299", "inputs": [ "l:36", "l:300" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:450" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:300", "inputs": [ "l:301" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "ed95422d29540bfc7f36e115c60ffa7a8494973d7cfe886d9611f90169997399", "shape": [ 200, 17 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "source_nodes": [ "s:451" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:301", "inputs": [ "l:302", "l:302", "l:303" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:452" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 36 }, { "annotation": null, "id": "l:302", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ], "values": [ 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:453" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:303", "inputs": [ "l:304" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "81e5bd2ad736b785c26f8ee5ad9ebc0d50450a71dfbb241759a0d378dfca6685", "shape": [ 3, 2 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:454" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:304", "inputs": [ "l:305" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "515d1614aef9d13e7b1592189bacbc4129f9986f5af077d91435419ef5889d28", "shape": [ 9 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "8052157cb96bc0305ae47a1a67c53dc01499b9f7c175846e4687adf3addaf1d9", "shape": [ 9, 6 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "480376c6bf738a0227f2bbf2b3506b7cde209152c0ba9a9077e5527169eb292e", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "30e1bbfda7f7c8c9053a5476e4b927c8d24e47e1e4f277056cf7b9b29f266de8", "shape": [ 9, 3 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 9 }, "source_nodes": [ "s:455" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:305", "inputs": [ "l:306", "l:308" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:456" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "l:306", "inputs": [ "l:307" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "480376c6bf738a0227f2bbf2b3506b7cde209152c0ba9a9077e5527169eb292e", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3f34e6886f39376bb37c08c97f7c480d64bc1e54adb1b9083da6ff148def52c6", "shape": [ 100, 3 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "source_nodes": [ "s:457" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:307", "inputs": [ "l:214", "l:222" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:458" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:308", "inputs": [ "l:309" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "515d1614aef9d13e7b1592189bacbc4129f9986f5af077d91435419ef5889d28", "shape": [ 9 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "8052157cb96bc0305ae47a1a67c53dc01499b9f7c175846e4687adf3addaf1d9", "shape": [ 9, 6 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "480376c6bf738a0227f2bbf2b3506b7cde209152c0ba9a9077e5527169eb292e", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "30e1bbfda7f7c8c9053a5476e4b927c8d24e47e1e4f277056cf7b9b29f266de8", "shape": [ 9, 3 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 9 }, "source_nodes": [ "s:459" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:309", "inputs": [ "l:310", "l:312" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:460" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "l:310", "inputs": [ "l:311" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "480376c6bf738a0227f2bbf2b3506b7cde209152c0ba9a9077e5527169eb292e", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3f34e6886f39376bb37c08c97f7c480d64bc1e54adb1b9083da6ff148def52c6", "shape": [ 100, 3 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "source_nodes": [ "s:461" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:311", "inputs": [ "l:194", "l:184" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:462" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:312", "inputs": [ "l:313" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "480376c6bf738a0227f2bbf2b3506b7cde209152c0ba9a9077e5527169eb292e", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3f34e6886f39376bb37c08c97f7c480d64bc1e54adb1b9083da6ff148def52c6", "shape": [ 100, 3 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "source_nodes": [ "s:463" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:313", "inputs": [ "l:83", "l:135" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:464" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:314", "inputs": [ "l:315" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:465" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:315", "inputs": [ "l:62", "l:68" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:466" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:316", "inputs": [ "l:317" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:467" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:317", "inputs": [ "l:71", "l:318" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:468" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:318", "inputs": [ "l:319" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "ed95422d29540bfc7f36e115c60ffa7a8494973d7cfe886d9611f90169997399", "shape": [ 200, 17 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "source_nodes": [ "s:469" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:319", "inputs": [ "l:262", "l:265", "l:320" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:470" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 36 }, { "annotation": null, "id": "l:320", "inputs": [ "l:321" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "81e5bd2ad736b785c26f8ee5ad9ebc0d50450a71dfbb241759a0d378dfca6685", "shape": [ 3, 2 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:471" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:321", "inputs": [ "l:322" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "515d1614aef9d13e7b1592189bacbc4129f9986f5af077d91435419ef5889d28", "shape": [ 9 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "8052157cb96bc0305ae47a1a67c53dc01499b9f7c175846e4687adf3addaf1d9", "shape": [ 9, 6 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "480376c6bf738a0227f2bbf2b3506b7cde209152c0ba9a9077e5527169eb292e", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "30e1bbfda7f7c8c9053a5476e4b927c8d24e47e1e4f277056cf7b9b29f266de8", "shape": [ 9, 3 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 9 }, "source_nodes": [ "s:472" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:322", "inputs": [ "l:205", "l:234" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:473" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "l:323", "inputs": [ "l:324" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0a981cf047d1413d4048a8918c40e5341ed3c1e754371a27df6dbebbc4041fc9", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:476" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:324", "inputs": [ "l:325" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f6bb1294da2f78cd935b01c7656280df5eaa0439e9d97bc03775825a41a508e4", "shape": [ 4, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:477" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 4.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:325", "inputs": [ "l:272", "l:273", "l:294", "l:326" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:478" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "l:326", "inputs": [ "l:327" ], "name": "fused_add_const_multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:479" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:327", "inputs": [ "l:328" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d02095ef9169de77f0d4bcaefe3494a01a6cb5303d8f16874d410303fff7354a", "shape": [ 1, 17 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "source_nodes": [ "s:481" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:328", "inputs": [ "l:329" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9de7dcac2491cd60a9a20fa1b13cbaa6a2ae2547b251a17e2001b747a444d46d", "shape": [ 51, 17 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:482" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:329", "inputs": [ "l:330", "l:337", "l:375" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:483" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 51 }, { "annotation": null, "id": "l:330", "inputs": [ "l:331" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:484" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:331", "inputs": [ "l:36", "l:332" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:485" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:332", "inputs": [ "l:333" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "ed95422d29540bfc7f36e115c60ffa7a8494973d7cfe886d9611f90169997399", "shape": [ 200, 17 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "source_nodes": [ "s:486" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:333", "inputs": [ "l:83", "l:135", "l:334" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:487" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 36 }, { "annotation": null, "id": "l:334", "inputs": [ "l:335" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "81e5bd2ad736b785c26f8ee5ad9ebc0d50450a71dfbb241759a0d378dfca6685", "shape": [ 3, 2 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:488" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:335", "inputs": [ "l:336" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "515d1614aef9d13e7b1592189bacbc4129f9986f5af077d91435419ef5889d28", "shape": [ 9 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "8052157cb96bc0305ae47a1a67c53dc01499b9f7c175846e4687adf3addaf1d9", "shape": [ 9, 6 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "480376c6bf738a0227f2bbf2b3506b7cde209152c0ba9a9077e5527169eb292e", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "30e1bbfda7f7c8c9053a5476e4b927c8d24e47e1e4f277056cf7b9b29f266de8", "shape": [ 9, 3 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 9 }, "source_nodes": [ "s:489" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:336", "inputs": [ "l:306", "l:310" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:490" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "l:337", "inputs": [ "l:338" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:491" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:338", "inputs": [ "l:62", "l:339" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:492" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:339", "inputs": [ "l:340" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "466a4cf2f6f4299d9cc7fd314c658f94d7a2a1dbe112cb598d2ea23fba6d9a08", "shape": [ 200, 17 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "source_nodes": [ "s:493" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:340", "inputs": [ "l:341", "l:358", "l:360" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:494" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 36 }, { "annotation": null, "id": "l:341", "inputs": [ "l:342" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "3422cbe5c1e8b5b1e016e89ff3387bd4290361fdb17782c50e53c187cf575c5d", "shape": [ 34, 35 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e0ebe36501a3a3bd49b9c2eca7f757e673188e21386e2ef140b112922c14d71b", "shape": [ 34, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7bd2d0249f58c2a4d46659f07872f284b01435b9afd6b0e16d9551cfac29acb6", "shape": [ 34, 35 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 34 }, "source_nodes": [ "s:495" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:342", "inputs": [ "l:343", "l:83", "l:135" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:496" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 35 }, { "annotation": null, "id": "l:343", "inputs": [ "l:344" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "25870a3818cab31267591ff755d69e74c8435d6e091e4ff7c8ca9945f2f3285c", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:497" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:344", "inputs": [ "l:345" ], "name": "cmp_score", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9e115a07ab905736a35a795d4e6bc3563c8cb99ceb53c43840d9195037bc779c", "shape": [ 6, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:498" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -13.0565631015625, "lower_unbounded": false, "upper": 13.0565631015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:345", "inputs": [ "l:346", "l:349", "l:350", "l:353", "l:354", "l:357" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:499" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "l:346", "inputs": [ "l:347" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "28c4f7df3425ac5087c2632d76786a4afaaf06906eeafb89d8d36ba650e3f24b", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "aced851223ade84128237ff20c2acca7ac1102b3d2dfea9f7fcfb3eef3457a01", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:500" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.00217550390625, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:347", "inputs": [ "l:348" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "64e48d62679d2010a89384ca9bac6ee4bcddfdfb5c838dc5aef8b4e8a40fbec8", "shape": [ 100, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "source_nodes": [ "s:501" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0, hi=1.0))" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:348", "inputs": [ "l:83", "l:135" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:502" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:349", "inputs": [ "l:347" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "25870a3818cab31267591ff755d69e74c8435d6e091e4ff7c8ca9945f2f3285c", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "a1f63f4bc9d1e1abb868bb7c396f4b79db414fdabb2519e660a356486e507089", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:503" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.00217550390625, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:350", "inputs": [ "l:351" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "28c4f7df3425ac5087c2632d76786a4afaaf06906eeafb89d8d36ba650e3f24b", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "aced851223ade84128237ff20c2acca7ac1102b3d2dfea9f7fcfb3eef3457a01", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:504" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.00217550390625, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:351", "inputs": [ "l:352" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "64e48d62679d2010a89384ca9bac6ee4bcddfdfb5c838dc5aef8b4e8a40fbec8", "shape": [ 100, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "source_nodes": [ "s:505" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0, hi=1.0))" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:352", "inputs": [ "l:194", "l:184" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:506" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:353", "inputs": [ "l:351" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "25870a3818cab31267591ff755d69e74c8435d6e091e4ff7c8ca9945f2f3285c", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "a1f63f4bc9d1e1abb868bb7c396f4b79db414fdabb2519e660a356486e507089", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:507" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.00217550390625, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:354", "inputs": [ "l:355" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "28c4f7df3425ac5087c2632d76786a4afaaf06906eeafb89d8d36ba650e3f24b", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "aced851223ade84128237ff20c2acca7ac1102b3d2dfea9f7fcfb3eef3457a01", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:508" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.00217550390625, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:355", "inputs": [ "l:356" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "64e48d62679d2010a89384ca9bac6ee4bcddfdfb5c838dc5aef8b4e8a40fbec8", "shape": [ 100, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "source_nodes": [ "s:509" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0, hi=1.0))" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:356", "inputs": [ "l:214", "l:222" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:510" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:357", "inputs": [ "l:355" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "25870a3818cab31267591ff755d69e74c8435d6e091e4ff7c8ca9945f2f3285c", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "a1f63f4bc9d1e1abb868bb7c396f4b79db414fdabb2519e660a356486e507089", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:511" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.00217550390625, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:358", "inputs": [ "l:359" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "3422cbe5c1e8b5b1e016e89ff3387bd4290361fdb17782c50e53c187cf575c5d", "shape": [ 34, 35 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e0ebe36501a3a3bd49b9c2eca7f757e673188e21386e2ef140b112922c14d71b", "shape": [ 34, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7bd2d0249f58c2a4d46659f07872f284b01435b9afd6b0e16d9551cfac29acb6", "shape": [ 34, 35 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 34 }, "source_nodes": [ "s:512" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:359", "inputs": [ "l:343", "l:135", "l:83" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:513" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 35 }, { "annotation": null, "id": "l:360", "inputs": [ "l:361" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "81e5bd2ad736b785c26f8ee5ad9ebc0d50450a71dfbb241759a0d378dfca6685", "shape": [ 3, 2 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:514" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:361", "inputs": [ "l:362" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "515d1614aef9d13e7b1592189bacbc4129f9986f5af077d91435419ef5889d28", "shape": [ 9 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "8052157cb96bc0305ae47a1a67c53dc01499b9f7c175846e4687adf3addaf1d9", "shape": [ 9, 6 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "480376c6bf738a0227f2bbf2b3506b7cde209152c0ba9a9077e5527169eb292e", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "30e1bbfda7f7c8c9053a5476e4b927c8d24e47e1e4f277056cf7b9b29f266de8", "shape": [ 9, 3 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 9 }, "source_nodes": [ "s:515" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:362", "inputs": [ "l:363", "l:369" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:516" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "l:363", "inputs": [ "l:364" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "480376c6bf738a0227f2bbf2b3506b7cde209152c0ba9a9077e5527169eb292e", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "51206ad98c150026d0995579546dd690743e9afd4918c78cb6e674f13df9da5b", "shape": [ 100, 3 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "source_nodes": [ "s:517" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:364", "inputs": [ "l:365", "l:367" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:518" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:365", "inputs": [ "l:366" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "3422cbe5c1e8b5b1e016e89ff3387bd4290361fdb17782c50e53c187cf575c5d", "shape": [ 34, 35 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e0ebe36501a3a3bd49b9c2eca7f757e673188e21386e2ef140b112922c14d71b", "shape": [ 34, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7bd2d0249f58c2a4d46659f07872f284b01435b9afd6b0e16d9551cfac29acb6", "shape": [ 34, 35 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 34 }, "source_nodes": [ "s:519" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:366", "inputs": [ "l:343", "l:214", "l:222" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:520" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 35 }, { "annotation": null, "id": "l:367", "inputs": [ "l:368" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "3422cbe5c1e8b5b1e016e89ff3387bd4290361fdb17782c50e53c187cf575c5d", "shape": [ 34, 35 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e0ebe36501a3a3bd49b9c2eca7f757e673188e21386e2ef140b112922c14d71b", "shape": [ 34, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7bd2d0249f58c2a4d46659f07872f284b01435b9afd6b0e16d9551cfac29acb6", "shape": [ 34, 35 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 34 }, "source_nodes": [ "s:521" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:368", "inputs": [ "l:343", "l:222", "l:214" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:522" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 35 }, { "annotation": null, "id": "l:369", "inputs": [ "l:370" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "480376c6bf738a0227f2bbf2b3506b7cde209152c0ba9a9077e5527169eb292e", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "51206ad98c150026d0995579546dd690743e9afd4918c78cb6e674f13df9da5b", "shape": [ 100, 3 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "source_nodes": [ "s:523" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "l:370", "inputs": [ "l:371", "l:373" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:524" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "l:371", "inputs": [ "l:372" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "3422cbe5c1e8b5b1e016e89ff3387bd4290361fdb17782c50e53c187cf575c5d", "shape": [ 34, 35 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e0ebe36501a3a3bd49b9c2eca7f757e673188e21386e2ef140b112922c14d71b", "shape": [ 34, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7bd2d0249f58c2a4d46659f07872f284b01435b9afd6b0e16d9551cfac29acb6", "shape": [ 34, 35 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 34 }, "source_nodes": [ "s:525" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:372", "inputs": [ "l:343", "l:194", "l:184" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:526" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 35 }, { "annotation": null, "id": "l:373", "inputs": [ "l:374" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "3422cbe5c1e8b5b1e016e89ff3387bd4290361fdb17782c50e53c187cf575c5d", "shape": [ 34, 35 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e0ebe36501a3a3bd49b9c2eca7f757e673188e21386e2ef140b112922c14d71b", "shape": [ 34, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7bd2d0249f58c2a4d46659f07872f284b01435b9afd6b0e16d9551cfac29acb6", "shape": [ 34, 35 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 34 }, "source_nodes": [ "s:527" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:374", "inputs": [ "l:343", "l:184", "l:194" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:528" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 35 }, { "annotation": null, "id": "l:375", "inputs": [ "l:376" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:529" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:376", "inputs": [ "l:71", "l:377" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:530" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:377", "inputs": [ "l:378" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "ed95422d29540bfc7f36e115c60ffa7a8494973d7cfe886d9611f90169997399", "shape": [ 200, 17 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "source_nodes": [ "s:531" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:378", "inputs": [ "l:248", "l:257", "l:379" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:532" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 36 }, { "annotation": null, "id": "l:379", "inputs": [ "l:288" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "81e5bd2ad736b785c26f8ee5ad9ebc0d50450a71dfbb241759a0d378dfca6685", "shape": [ 3, 2 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:533" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:380", "inputs": [ "l:381" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "db8d1a556ec9527f63930661298f4b8607cfcecb3c364235751acfef1ec8ed1a", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:536" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:381", "inputs": [ "l:382" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "6ffd58417e23083d07218fa4d272fa091844ae58d9af49caa33e453e2e8dbaf8", "shape": [ 5, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:537" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.010877519531249935, "lower_unbounded": false, "upper": 5.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:382", "inputs": [ "l:272", "l:273", "l:294", "l:326", "l:383" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 5 }, { "annotation": null, "id": "l:383", "inputs": [ "l:384" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d02095ef9169de77f0d4bcaefe3494a01a6cb5303d8f16874d410303fff7354a", "shape": [ 1, 17 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "source_nodes": [ "s:541" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:384", "inputs": [ "l:385" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9de7dcac2491cd60a9a20fa1b13cbaa6a2ae2547b251a17e2001b747a444d46d", "shape": [ 51, 17 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:542" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:385", "inputs": [ "l:386", "l:391", "l:396" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:543" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 51 }, { "annotation": null, "id": "l:386", "inputs": [ "l:387" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:544" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:387", "inputs": [ "l:36", "l:388" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:545" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:388", "inputs": [ "l:389" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "ed95422d29540bfc7f36e115c60ffa7a8494973d7cfe886d9611f90169997399", "shape": [ 200, 17 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "source_nodes": [ "s:546" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:389", "inputs": [ "l:194", "l:184", "l:390" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:547" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 36 }, { "annotation": null, "id": "l:390", "inputs": [ "l:306" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "81e5bd2ad736b785c26f8ee5ad9ebc0d50450a71dfbb241759a0d378dfca6685", "shape": [ 3, 2 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:548" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:391", "inputs": [ "l:392" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:549" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:392", "inputs": [ "l:62", "l:393" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:550" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:393", "inputs": [ "l:394" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "466a4cf2f6f4299d9cc7fd314c658f94d7a2a1dbe112cb598d2ea23fba6d9a08", "shape": [ 200, 17 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "source_nodes": [ "s:551" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:394", "inputs": [ "l:371", "l:373", "l:395" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:552" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 36 }, { "annotation": null, "id": "l:395", "inputs": [ "l:363" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "81e5bd2ad736b785c26f8ee5ad9ebc0d50450a71dfbb241759a0d378dfca6685", "shape": [ 3, 2 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:553" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:396", "inputs": [ "l:397" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:554" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:397", "inputs": [ "l:71", "l:398" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:555" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:398", "inputs": [ "l:399" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "ed95422d29540bfc7f36e115c60ffa7a8494973d7cfe886d9611f90169997399", "shape": [ 200, 17 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "source_nodes": [ "s:556" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:399", "inputs": [ "l:238", "l:245", "l:400" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:557" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 36 }, { "annotation": null, "id": "l:400", "inputs": [ "l:205" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "81e5bd2ad736b785c26f8ee5ad9ebc0d50450a71dfbb241759a0d378dfca6685", "shape": [ 3, 2 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:558" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:401", "inputs": [ "l:402" ], "name": "fused_fused_add_const_multiply_const_multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "ea2845900b5856c9bf354b1aa9761b5aa6888e5ed61738fe9579ca42bc0f6054", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "ea2845900b5856c9bf354b1aa9761b5aa6888e5ed61738fe9579ca42bc0f6054", "shape": [ 1, 1 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:559" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.013053023437500144, "lower_unbounded": false, "upper": 6.0130530234375, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:402", "inputs": [ "l:403" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:562" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:403", "inputs": [ "l:62", "l:404" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:564" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:404", "inputs": [ "l:343" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "f41491c367f49e7206c45724651464310fcb7d17e3059779d1c95aa901f3fce1", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:565" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:405", "inputs": [ "l:32", "l:275", "l:296", "l:328", "l:384", "l:406", "l:423", "l:423", "l:423", "l:423", "l:423", "l:423" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:567" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "l:406", "inputs": [ "l:407" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9de7dcac2491cd60a9a20fa1b13cbaa6a2ae2547b251a17e2001b747a444d46d", "shape": [ 51, 17 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:568" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:407", "inputs": [ "l:408", "l:413", "l:418" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:569" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 51 }, { "annotation": null, "id": "l:408", "inputs": [ "l:409" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:570" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:409", "inputs": [ "l:36", "l:410" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:571" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:410", "inputs": [ "l:411" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "ed95422d29540bfc7f36e115c60ffa7a8494973d7cfe886d9611f90169997399", "shape": [ 200, 17 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "source_nodes": [ "s:572" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:411", "inputs": [ "l:214", "l:222", "l:412" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:573" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 36 }, { "annotation": null, "id": "l:412", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "434b26042aff3fb844a4c4c6be0d81a079b0ce84cfb8190679024404e5dc4822", "shape": [ 2 ], "values": [ 1.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:574" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:413", "inputs": [ "l:414" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:575" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:414", "inputs": [ "l:62", "l:415" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:576" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:415", "inputs": [ "l:416" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "466a4cf2f6f4299d9cc7fd314c658f94d7a2a1dbe112cb598d2ea23fba6d9a08", "shape": [ 200, 17 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "source_nodes": [ "s:577" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:416", "inputs": [ "l:365", "l:367", "l:417" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:578" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 36 }, { "annotation": null, "id": "l:417", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "434b26042aff3fb844a4c4c6be0d81a079b0ce84cfb8190679024404e5dc4822", "shape": [ 2 ], "values": [ 1.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:579" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:418", "inputs": [ "l:419" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:580" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:419", "inputs": [ "l:71", "l:420" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:581" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:420", "inputs": [ "l:421" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "ed95422d29540bfc7f36e115c60ffa7a8494973d7cfe886d9611f90169997399", "shape": [ 200, 17 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "source_nodes": [ "s:582" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:421", "inputs": [ "l:207", "l:231", "l:422" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:583" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 36 }, { "annotation": null, "id": "l:422", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "434b26042aff3fb844a4c4c6be0d81a079b0ce84cfb8190679024404e5dc4822", "shape": [ 2 ], "values": [ 1.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:584" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:423", "inputs": [], "name": "remove_leading_0s_sign", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "4ef898bcd05c1b46ff5a572b9c0da67abd133f76ada08aa6e708d5322587027c", "shape": [ 17 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:585" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:424", "inputs": [ "l:425" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:586" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=33.45360234820311))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 33.45360234820311, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 33.45360234820311, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:425", "inputs": [ "l:426", "l:430" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:587" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 33.45360234820311, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:426", "inputs": [ "l:427" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:588" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:427", "inputs": [ "l:428", "l:429" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:590" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:428", "inputs": [ "l:17" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "28c4f7df3425ac5087c2632d76786a4afaaf06906eeafb89d8d36ba650e3f24b", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:591" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:429", "inputs": [ "l:17" ], "name": "fused_negate_compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "9bc792642836b330e7b77ccfcb99d0ea4176f98a05cc12439b285610ead907ae", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bedf28cd77bc6a0aa72157aa10d604f1203eb0409b546f9a088311112009bb5b", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:592" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:430", "inputs": [ "l:431" ], "name": "remove_leading_0s_sum", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d7c028b7eda356619d577d7e3dfef5da2267e2c58f989f7adcb5316c4793dd4c", "shape": [ 204, 17 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:594" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 33.45360234820311, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:431", "inputs": [ "l:432" ], "name": "broadcast_select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2c93ccfa935dc7b48ae36ca9b98d2c3e3e50962957d9f3b464865f5c129d7072", "shape": [ 204, 216 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "08720f852c879614691b2c33f2980888e7cdf29871f0a4e9e60e7446758bfc2e", "shape": [ 204, 204 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7e26b5245da34ef2eb9e8e57159215ac9a011c4de6a4aa42cdbfb51763b7de8f", "shape": [ 204, 216 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 204 }, "source_nodes": [ "s:595" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0149999999999997))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "l:432", "inputs": [ "l:26", "l:433" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:596" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 216 }, { "annotation": null, "id": "l:433", "inputs": [ "l:275", "l:296", "l:328", "l:384", "l:406", "l:434", "l:32", "l:275", "l:296", "l:328", "l:384", "l:406" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:597" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "l:434", "inputs": [ "l:435" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9de7dcac2491cd60a9a20fa1b13cbaa6a2ae2547b251a17e2001b747a444d46d", "shape": [ 51, 17 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:598" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:435", "inputs": [ "l:436", "l:439", "l:442" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:599" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 51 }, { "annotation": null, "id": "l:436", "inputs": [ "l:437" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:600" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:437", "inputs": [ "l:36", "l:438" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:601" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:438", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "0bbff7d224d8ac368f7ae20a16cb4ecb95004d3ca9ad72eb7433632c0fab6028", "shape": [ 17 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:602" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:439", "inputs": [ "l:440" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:603" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:440", "inputs": [ "l:62", "l:441" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:604" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:441", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "0bbff7d224d8ac368f7ae20a16cb4ecb95004d3ca9ad72eb7433632c0fab6028", "shape": [ 17 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:605" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:442", "inputs": [ "l:443" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:606" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:443", "inputs": [ "l:71", "l:444" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:607" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:444", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "0bbff7d224d8ac368f7ae20a16cb4ecb95004d3ca9ad72eb7433632c0fab6028", "shape": [ 17 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:608" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:445", "inputs": [ "l:446" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:609" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=27.371129193984366))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 27.371129193984366, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 27.371129193984366, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:446", "inputs": [ "l:447", "l:451" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:610" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 27.371129193984366, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:447", "inputs": [ "l:448" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:611" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:448", "inputs": [ "l:449", "l:450" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:613" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:449", "inputs": [ "l:17" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "eb9f1ff3e4fa10d12a6ae64cce02175007e98fff40f70e2ba8fdbf8f5a123795", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:614" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:450", "inputs": [ "l:17" ], "name": "fused_negate_compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "ed53dbef9c852a8229e4b386e20627c3a89599c7583ec50379471fb4415bc5b3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bedf28cd77bc6a0aa72157aa10d604f1203eb0409b546f9a088311112009bb5b", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:615" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:451", "inputs": [ "l:452" ], "name": "remove_leading_0s_sum", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d7c028b7eda356619d577d7e3dfef5da2267e2c58f989f7adcb5316c4793dd4c", "shape": [ 204, 17 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:617" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 27.371129193984366, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:452", "inputs": [ "l:453" ], "name": "broadcast_select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2c93ccfa935dc7b48ae36ca9b98d2c3e3e50962957d9f3b464865f5c129d7072", "shape": [ 204, 216 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "08720f852c879614691b2c33f2980888e7cdf29871f0a4e9e60e7446758bfc2e", "shape": [ 204, 204 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7e26b5245da34ef2eb9e8e57159215ac9a011c4de6a4aa42cdbfb51763b7de8f", "shape": [ 204, 216 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 204 }, "source_nodes": [ "s:618" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0149999999999997))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "l:453", "inputs": [ "l:26", "l:454" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:619" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 216 }, { "annotation": null, "id": "l:454", "inputs": [ "l:296", "l:328", "l:384", "l:406", "l:434", "l:455", "l:275", "l:296", "l:328", "l:384", "l:406", "l:434" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:620" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "l:455", "inputs": [ "l:456" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9de7dcac2491cd60a9a20fa1b13cbaa6a2ae2547b251a17e2001b747a444d46d", "shape": [ 51, 17 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:621" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:456", "inputs": [ "l:457", "l:459", "l:461" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:622" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 51 }, { "annotation": null, "id": "l:457", "inputs": [ "l:458" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:623" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:458", "inputs": [ "l:36", "l:438" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:624" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:459", "inputs": [ "l:460" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:625" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:460", "inputs": [ "l:62", "l:441" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:626" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:461", "inputs": [ "l:462" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:627" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:462", "inputs": [ "l:71", "l:444" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:628" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:463", "inputs": [ "l:464" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:629" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=21.28865603976562))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 21.28865603976562, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 21.28865603976562, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:464", "inputs": [ "l:465", "l:469" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:630" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 21.28865603976562, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:465", "inputs": [ "l:466" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:631" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:466", "inputs": [ "l:467", "l:468" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:633" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:467", "inputs": [ "l:17" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1cc95150dbcdc695d9fe82393ccc24788e0bbdb47c092c4100228da7b850f0b9", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:634" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:468", "inputs": [ "l:17" ], "name": "fused_negate_compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "9dfa5cc0a4093ad092e8327b9861cf4f1595c50fa4b3f12efc578b78fe5afdd2", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bedf28cd77bc6a0aa72157aa10d604f1203eb0409b546f9a088311112009bb5b", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:635" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:469", "inputs": [ "l:470" ], "name": "remove_leading_0s_sum", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d7c028b7eda356619d577d7e3dfef5da2267e2c58f989f7adcb5316c4793dd4c", "shape": [ 204, 17 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:637" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 21.28865603976562, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:470", "inputs": [ "l:471" ], "name": "broadcast_select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2c93ccfa935dc7b48ae36ca9b98d2c3e3e50962957d9f3b464865f5c129d7072", "shape": [ 204, 216 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "08720f852c879614691b2c33f2980888e7cdf29871f0a4e9e60e7446758bfc2e", "shape": [ 204, 204 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7e26b5245da34ef2eb9e8e57159215ac9a011c4de6a4aa42cdbfb51763b7de8f", "shape": [ 204, 216 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 204 }, "source_nodes": [ "s:638" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0149999999999997))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "l:471", "inputs": [ "l:26", "l:472" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:639" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 216 }, { "annotation": null, "id": "l:472", "inputs": [ "l:328", "l:384", "l:406", "l:434", "l:455", "l:455", "l:296", "l:328", "l:384", "l:406", "l:434", "l:455" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:640" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "l:473", "inputs": [ "l:474" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:641" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=21.288656039765623))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 21.288656039765623, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 21.288656039765623, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:474", "inputs": [ "l:475", "l:479" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:642" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 21.288656039765623, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:475", "inputs": [ "l:476" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:643" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:476", "inputs": [ "l:477", "l:478" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:645" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:477", "inputs": [ "l:17" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0a981cf047d1413d4048a8918c40e5341ed3c1e754371a27df6dbebbc4041fc9", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:646" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:478", "inputs": [ "l:17" ], "name": "fused_negate_compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "25eab693bdf7c6ad20a03812dd32b8a7ba27dab587478a5dc1c8b657174ae97d", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bedf28cd77bc6a0aa72157aa10d604f1203eb0409b546f9a088311112009bb5b", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:647" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:479", "inputs": [ "l:480" ], "name": "remove_leading_0s_sum", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d7c028b7eda356619d577d7e3dfef5da2267e2c58f989f7adcb5316c4793dd4c", "shape": [ 204, 17 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:649" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 21.288656039765623, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:480", "inputs": [ "l:481" ], "name": "broadcast_select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2c93ccfa935dc7b48ae36ca9b98d2c3e3e50962957d9f3b464865f5c129d7072", "shape": [ 204, 216 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "08720f852c879614691b2c33f2980888e7cdf29871f0a4e9e60e7446758bfc2e", "shape": [ 204, 204 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7e26b5245da34ef2eb9e8e57159215ac9a011c4de6a4aa42cdbfb51763b7de8f", "shape": [ 204, 216 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 204 }, "source_nodes": [ "s:650" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0149999999999997))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "l:481", "inputs": [ "l:26", "l:482" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:651" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 216 }, { "annotation": null, "id": "l:482", "inputs": [ "l:384", "l:406", "l:434", "l:455", "l:455", "l:455", "l:328", "l:384", "l:406", "l:434", "l:455", "l:455" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:652" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "l:483", "inputs": [ "l:484" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:653" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=27.371129193984373))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 27.371129193984373, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 27.371129193984373, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:484", "inputs": [ "l:485", "l:489" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:654" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 27.371129193984373, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:485", "inputs": [ "l:486" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:655" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:486", "inputs": [ "l:487", "l:488" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:657" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:487", "inputs": [ "l:17" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "db8d1a556ec9527f63930661298f4b8607cfcecb3c364235751acfef1ec8ed1a", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:658" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:488", "inputs": [ "l:17" ], "name": "fused_negate_compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "64f403ba374325ca540cb27c0375a0ea1dc20ecaeb654fcbe8bea2b756826dab", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bedf28cd77bc6a0aa72157aa10d604f1203eb0409b546f9a088311112009bb5b", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:659" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:489", "inputs": [ "l:490" ], "name": "remove_leading_0s_sum", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d7c028b7eda356619d577d7e3dfef5da2267e2c58f989f7adcb5316c4793dd4c", "shape": [ 204, 17 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:661" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 27.371129193984373, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:490", "inputs": [ "l:491" ], "name": "broadcast_select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2c93ccfa935dc7b48ae36ca9b98d2c3e3e50962957d9f3b464865f5c129d7072", "shape": [ 204, 216 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "08720f852c879614691b2c33f2980888e7cdf29871f0a4e9e60e7446758bfc2e", "shape": [ 204, 204 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7e26b5245da34ef2eb9e8e57159215ac9a011c4de6a4aa42cdbfb51763b7de8f", "shape": [ 204, 216 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 204 }, "source_nodes": [ "s:662" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0149999999999997))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "l:491", "inputs": [ "l:26", "l:492" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:663" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 216 }, { "annotation": null, "id": "l:492", "inputs": [ "l:406", "l:434", "l:455", "l:455", "l:455", "l:455", "l:384", "l:406", "l:434", "l:455", "l:455", "l:455" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:664" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "l:493", "inputs": [ "l:494" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:665" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=33.45360234820312))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 33.45360234820312, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 33.45360234820312, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:494", "inputs": [ "l:495", "l:499" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:666" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 33.45360234820312, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:495", "inputs": [ "l:496" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:667" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:496", "inputs": [ "l:497", "l:498" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:669" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:497", "inputs": [ "l:17" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b8bb7a27e55e02d67b43582292d3134e5715248dc3f4a54dac7e1c71519551b5", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:670" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:498", "inputs": [ "l:17" ], "name": "fused_negate_compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df1b487fa89261e6181b4f1197a2248f295f75619fa52d6c77c2a438a9f5862d", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bedf28cd77bc6a0aa72157aa10d604f1203eb0409b546f9a088311112009bb5b", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:671" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:499", "inputs": [ "l:500" ], "name": "remove_leading_0s_sum", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d7c028b7eda356619d577d7e3dfef5da2267e2c58f989f7adcb5316c4793dd4c", "shape": [ 204, 17 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:673" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 33.45360234820312, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:500", "inputs": [ "l:501" ], "name": "broadcast_select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2c93ccfa935dc7b48ae36ca9b98d2c3e3e50962957d9f3b464865f5c129d7072", "shape": [ 204, 216 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "08720f852c879614691b2c33f2980888e7cdf29871f0a4e9e60e7446758bfc2e", "shape": [ 204, 204 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7e26b5245da34ef2eb9e8e57159215ac9a011c4de6a4aa42cdbfb51763b7de8f", "shape": [ 204, 216 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 204 }, "source_nodes": [ "s:674" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0149999999999997))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "l:501", "inputs": [ "l:26", "l:502" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:675" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 216 }, { "annotation": null, "id": "l:502", "inputs": [ "l:434", "l:455", "l:455", "l:455", "l:455", "l:455", "l:406", "l:434", "l:455", "l:455", "l:455", "l:455" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:676" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "l:503", "inputs": [ "l:504" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "source_nodes": [ "s:677" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=36.49483892531249))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 36.49483892531249, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 36.49483892531249, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:504", "inputs": [ "l:505", "l:509" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:678" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 36.49483892531249, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "l:505", "inputs": [ "l:506" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bcf7ac4b1f65963b5f231e5ba36e318f9b96c792cfcbe5979bae308b273f48e5", "shape": [ 2, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:679" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:506", "inputs": [ "l:507", "l:508" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:681" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "l:507", "inputs": [ "l:17" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "fcc789a616d690b13c0ce2bb32111dec233ff36515f19e597b74cfe3b5615bcd", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:682" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:508", "inputs": [ "l:17" ], "name": "fused_negate_compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "ee1c1e99aec431b3878503f437c2d6ed686bab105f28857153577a62843be75e", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "bedf28cd77bc6a0aa72157aa10d604f1203eb0409b546f9a088311112009bb5b", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "source_nodes": [ "s:683" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "l:509", "inputs": [ "l:510" ], "name": "remove_leading_0s_sum", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d7c028b7eda356619d577d7e3dfef5da2267e2c58f989f7adcb5316c4793dd4c", "shape": [ 204, 17 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:685" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 36.49483892531249, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "l:510", "inputs": [ "l:511" ], "name": "broadcast_select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2c93ccfa935dc7b48ae36ca9b98d2c3e3e50962957d9f3b464865f5c129d7072", "shape": [ 204, 216 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "08720f852c879614691b2c33f2980888e7cdf29871f0a4e9e60e7446758bfc2e", "shape": [ 204, 204 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7e26b5245da34ef2eb9e8e57159215ac9a011c4de6a4aa42cdbfb51763b7de8f", "shape": [ 204, 216 ] } ], "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 204 }, "source_nodes": [ "s:686" ], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0149999999999997))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "l:511", "inputs": [ "l:26", "l:512" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:687" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 216 }, { "annotation": null, "id": "l:512", "inputs": [ "l:455", "l:455", "l:455", "l:455", "l:455", "l:455", "l:434", "l:455", "l:455", "l:455", "l:455", "l:455" ], "name": null, "op": "Concatenate", "parameters": [], "scheduling_predecessors": [], "source_nodes": [ "s:688" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "l:513", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "bdc7bf3db562e39b656aed4e81ea8425eddc35662c76ad9ef2f006e36d6c43d0", "shape": [ 17 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0 ] } ], "scheduling_predecessors": [], "source_nodes": [ "s:689" ], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 } ], "root": "l:0", "sha256": "7c27dfefbc957fdc2f26afce75b6b63937c824a6e7be53e3e174930aedd4242c" }, "realization_map": [ { "lowered": "l:0", "source": "s:0", "status": "whole" }, { "lowered": "l:1", "source": "s:1", "status": "whole" }, { "lowered": "l:2", "source": "s:2", "status": "whole" }, { "lowered": "l:3", "source": "s:3", "status": "whole" }, { "lowered": "l:4", "source": "s:4", "status": "whole" }, { "lowered": "l:5", "source": "s:5", "status": "whole" }, { "lowered": "l:6", "source": "s:6", "status": "whole" }, { "lowered": "l:7", "source": "s:7", "status": "whole" }, { "lowered": "l:8", "source": "s:8", "status": "whole" }, { "lowered": "l:9", "source": "s:9", "status": "whole" }, { "lowered": "l:10", "source": "s:10", "status": "whole" }, { "lowered": "l:11", "source": "s:11", "status": "whole" }, { "lowered": "l:12", "source": "s:12", "status": "whole" }, { "lowered": "l:13", "source": "s:13", "status": "whole" }, { "lowered": "l:14", "source": "s:14", "status": "whole" }, { "source": "s:15", "status": "not_materialized" }, { "lowered": "l:15", "source": "s:16", "status": "whole" }, { "lowered": "l:16", "source": "s:17", "status": "whole" }, { "lowered": "l:17", "source": "s:18", "status": "whole" }, { "lowered": "l:18", "source": "s:19", "status": "whole" }, { "lowered": "l:19", "source": "s:20", "status": "whole" }, { "lowered": "l:20", "source": "s:21", "status": "whole" }, { "lowered": "l:21", "source": "s:22", "status": "whole" }, { "source": "s:23", "status": "not_materialized" }, { "lowered": "l:22", "source": "s:24", "status": "whole" }, { "source": "s:25", "status": "not_materialized" }, { "lowered": "l:23", "source": "s:26", "status": "whole" }, { "lowered": "l:24", "source": "s:27", "status": "whole" }, { "lowered": "l:25", "source": "s:28", "status": "whole" }, { "lowered": "l:26", "source": "s:29", "status": "whole" }, { "source": "s:30", "status": "not_materialized" }, { "lowered": "l:27", "source": "s:31", "status": "whole" }, { "lowered": "l:28", "source": "s:32", "status": "whole" }, { "source": "s:33", "status": "not_materialized" }, { "source": "s:34", "status": "not_materialized" }, { "source": "s:35", "status": "not_materialized" }, { "lowered": "l:30", "source": "s:36", "status": "whole" }, { "source": "s:37", "status": "not_materialized" }, { "source": "s:38", "status": "not_materialized" }, { "lowered": "l:272", "source": "s:39", "status": "whole" }, { "source": "s:40", "status": "not_materialized" }, { "lowered": "l:31", "source": "s:41", "status": "whole" }, { "lowered": "l:32", "source": "s:42", "status": "whole" }, { "lowered": "l:33", "source": "s:43", "status": "whole" }, { "lowered": "l:34", "source": "s:44", "status": "whole" }, { "lowered": "l:35", "source": "s:45", "status": "whole" }, { "lowered": "l:36", "source": "s:46", "status": "whole" }, { "lowered": "l:37", "source": "s:47", "status": "whole" }, { "lowered": "l:38", "source": "s:48", "status": "whole" }, { "lowered": "l:39", "source": "s:49", "status": "whole" }, { "lowered": "l:40", "source": "s:50", "status": "whole" }, { "lowered": "l:41", "source": "s:51", "status": "whole" }, { "source": "s:52", "status": "not_materialized" }, { "lowered": "l:42", "source": "s:53", "status": "whole" }, { "lowered": "l:43", "source": "s:54", "status": "whole" }, { "source": "s:55", "status": "not_materialized" }, { "lowered": "l:44", "source": "s:56", "status": "whole" }, { "lowered": "l:45", "source": "s:57", "status": "whole" }, { "lowered": "l:46", "source": "s:58", "status": "whole" }, { "lowered": "l:47", "source": "s:59", "status": "whole" }, { "lowered": "l:48", "source": "s:60", "status": "whole" }, { "lowered": "l:49", "source": "s:61", "status": "whole" }, { "lowered": "l:50", "source": "s:62", "status": "whole" }, { "lowered": "l:51", "source": "s:63", "status": "whole" }, { "lowered": "l:52", "source": "s:64", "status": "whole" }, { "lowered": "l:53", "source": "s:65", "status": "whole" }, { "lowered": "l:54", "source": "s:66", "status": "whole" }, { "lowered": "l:55", "source": "s:67", "status": "whole" }, { "lowered": "l:56", "source": "s:68", "status": "whole" }, { "lowered": "l:57", "source": "s:69", "status": "whole" }, { "lowered": "l:58", "source": "s:70", "status": "whole" }, { "lowered": "l:59", "source": "s:71", "status": "whole" }, { "lowered": "l:60", "source": "s:72", "status": "whole" }, { "lowered": "l:61", "source": "s:73", "status": "whole" }, { "lowered": "l:62", "source": "s:74", "status": "whole" }, { "lowered": "l:63", "source": "s:75", "status": "whole" }, { "lowered": "l:64", "source": "s:76", "status": "whole" }, { "lowered": "l:65", "source": "s:77", "status": "whole" }, { "lowered": "l:66", "source": "s:78", "status": "whole" }, { "lowered": "l:67", "source": "s:79", "status": "whole" }, { "lowered": "l:68", "source": "s:80", "status": "whole" }, { "lowered": "l:69", "source": "s:81", "status": "whole" }, { "lowered": "l:70", "source": "s:82", "status": "whole" }, { "lowered": "l:71", "source": "s:83", "status": "whole" }, { "lowered": "l:72", "source": "s:84", "status": "whole" }, { "lowered": "l:73", "source": "s:85", "status": "whole" }, { "lowered": "l:74", "source": "s:86", "status": "whole" }, { "lowered": "l:75", "source": "s:87", "status": "whole" }, { "lowered": "l:76", "source": "s:88", "status": "whole" }, { "lowered": "l:77", "source": "s:89", "status": "whole" }, { "lowered": "l:78", "source": "s:90", "status": "whole" }, { "lowered": "l:79", "source": "s:91", "status": "whole" }, { "source": "s:92", "status": "not_materialized" }, { "source": "s:93", "status": "not_materialized" }, { "source": "s:94", "status": "not_materialized" }, { "source": "s:95", "status": "not_materialized" }, { "lowered": "l:80", "source": "s:96", "status": "whole" }, { "source": "s:97", "status": "not_materialized" }, { "source": "s:98", "status": "not_materialized" }, { "source": "s:99", "status": "not_materialized" }, { "lowered": "l:81", "source": "s:100", "status": "whole" }, { "lowered": "l:82", "source": "s:101", "status": "whole" }, { "lowered": "l:83", "source": "s:102", "status": "whole" }, { "lowered": "l:84", "source": "s:103", "status": "whole" }, { "lowered": "l:85", "source": "s:104", "status": "whole" }, { "lowered": "l:86", "source": "s:105", "status": "whole" }, { "lowered": "l:87", "source": "s:106", "status": "whole" }, { "lowered": "l:88", "source": "s:107", "status": "whole" }, { "lowered": "l:89", "source": "s:108", "status": "whole" }, { "lowered": "l:90", "source": "s:109", "status": "whole" }, { "source": "s:110", "status": "not_materialized" }, { "source": "s:111", "status": "not_materialized" }, { "source": "s:112", "status": "not_materialized" }, { "source": "s:113", "status": "not_materialized" }, { "source": "s:114", "status": "not_materialized" }, { "source": "s:115", "status": "not_materialized" }, { "lowered": "l:91", "source": "s:116", "status": "whole" }, { "lowered": "l:92", "source": "s:117", "status": "whole" }, { "lowered": "l:93", "source": "s:118", "status": "whole" }, { "lowered": "l:94", "source": "s:119", "status": "whole" }, { "lowered": "l:95", "source": "s:120", "status": "whole" }, { "lowered": "l:96", "source": "s:121", "status": "whole" }, { "lowered": "l:97", "source": "s:122", "status": "whole" }, { "lowered": "l:98", "source": "s:123", "status": "whole" }, { "lowered": "l:99", "source": "s:124", "status": "whole" }, { "lowered": "l:100", "source": "s:125", "status": "whole" }, { "lowered": "l:101", "source": "s:126", "status": "whole" }, { "lowered": "l:102", "source": "s:127", "status": "whole" }, { "lowered": "l:103", "source": "s:128", "status": "whole" }, { "lowered": "l:104", "source": "s:129", "status": "whole" }, { "lowered": "l:105", "source": "s:130", "status": "whole" }, { "lowered": "l:106", "source": "s:131", "status": "whole" }, { "lowered": "l:107", "source": "s:132", "status": "whole" }, { "lowered": "l:108", "source": "s:133", "status": "whole" }, { "source": "s:134", "status": "not_materialized" }, { "source": "s:135", "status": "not_materialized" }, { "source": "s:136", "status": "not_materialized" }, { "source": "s:137", "status": "not_materialized" }, { "lowered": "l:109", "source": "s:138", "status": "whole" }, { "lowered": "l:110", "source": "s:139", "status": "whole" }, { "lowered": "l:111", "source": "s:140", "status": "whole" }, { "lowered": "l:112", "source": "s:141", "status": "whole" }, { "lowered": "l:113", "source": "s:142", "status": "whole" }, { "source": "s:143", "status": "not_materialized" }, { "lowered": "l:114", "source": "s:144", "status": "whole" }, { "lowered": "l:115", "source": "s:145", "status": "whole" }, { "lowered": "l:116", "source": "s:146", "status": "whole" }, { "lowered": "l:117", "source": "s:147", "status": "whole" }, { "lowered": "l:118", "source": "s:148", "status": "whole" }, { "lowered": "l:119", "source": "s:149", "status": "whole" }, { "lowered": "l:120", "source": "s:150", "status": "whole" }, { "lowered": "l:121", "source": "s:151", "status": "whole" }, { "lowered": "l:122", "source": "s:152", "status": "whole" }, { "lowered": "l:123", "source": "s:153", "status": "whole" }, { "source": "s:154", "status": "not_materialized" }, { "lowered": "l:124", "source": "s:155", "status": "whole" }, { "lowered": "l:125", "source": "s:156", "status": "whole" }, { "lowered": "l:126", "source": "s:157", "status": "whole" }, { "lowered": "l:127", "source": "s:158", "status": "whole" }, { "lowered": "l:128", "source": "s:159", "status": "whole" }, { "lowered": "l:129", "source": "s:160", "status": "whole" }, { "lowered": "l:130", "source": "s:161", "status": "whole" }, { "lowered": "l:131", "source": "s:162", "status": "whole" }, { "lowered": "l:132", "source": "s:163", "status": "whole" }, { "lowered": "l:133", "source": "s:164", "status": "whole" }, { "lowered": "l:134", "source": "s:165", "status": "whole" }, { "lowered": "l:135", "source": "s:166", "status": "whole" }, { "lowered": "l:136", "source": "s:167", "status": "whole" }, { "lowered": "l:137", "source": "s:168", "status": "whole" }, { "lowered": "l:138", "source": "s:169", "status": "whole" }, { "lowered": "l:139", "source": "s:170", "status": "whole" }, { "lowered": "l:140", "source": "s:171", "status": "whole" }, { "lowered": "l:141", "source": "s:172", "status": "whole" }, { "lowered": "l:142", "source": "s:173", "status": "whole" }, { "source": "s:174", "status": "not_materialized" }, { "source": "s:175", "status": "not_materialized" }, { "source": "s:176", "status": "not_materialized" }, { "source": "s:177", "status": "not_materialized" }, { "source": "s:178", "status": "not_materialized" }, { "source": "s:179", "status": "not_materialized" }, { "lowered": "l:143", "source": "s:180", "status": "whole" }, { "lowered": "l:144", "source": "s:181", "status": "whole" }, { "lowered": "l:145", "source": "s:182", "status": "whole" }, { "lowered": "l:146", "source": "s:183", "status": "whole" }, { "lowered": "l:147", "source": "s:184", "status": "whole" }, { "lowered": "l:148", "source": "s:185", "status": "whole" }, { "lowered": "l:149", "source": "s:186", "status": "whole" }, { "lowered": "l:150", "source": "s:187", "status": "whole" }, { "lowered": "l:151", "source": "s:188", "status": "whole" }, { "lowered": "l:152", "source": "s:189", "status": "whole" }, { "lowered": "l:153", "source": "s:190", "status": "whole" }, { "lowered": "l:154", "source": "s:191", "status": "whole" }, { "lowered": "l:155", "source": "s:192", "status": "whole" }, { "lowered": "l:156", "source": "s:193", "status": "whole" }, { "lowered": "l:157", "source": "s:194", "status": "whole" }, { "lowered": "l:158", "source": "s:195", "status": "whole" }, { "lowered": "l:159", "source": "s:196", "status": "whole" }, { "source": "s:197", "status": "not_materialized" }, { "source": "s:198", "status": "not_materialized" }, { "source": "s:199", "status": "not_materialized" }, { "source": "s:200", "status": "not_materialized" }, { "lowered": "l:160", "source": "s:201", "status": "whole" }, { "lowered": "l:161", "source": "s:202", "status": "whole" }, { "lowered": "l:162", "source": "s:203", "status": "whole" }, { "lowered": "l:163", "source": "s:204", "status": "whole" }, { "lowered": "l:164", "source": "s:205", "status": "whole" }, { "source": "s:206", "status": "not_materialized" }, { "lowered": "l:165", "source": "s:207", "status": "whole" }, { "lowered": "l:166", "source": "s:208", "status": "whole" }, { "lowered": "l:167", "source": "s:209", "status": "whole" }, { "source": "s:210", "status": "not_materialized" }, { "lowered": "l:168", "source": "s:211", "status": "whole" }, { "lowered": "l:169", "source": "s:212", "status": "whole" }, { "lowered": "l:170", "source": "s:213", "status": "whole" }, { "lowered": "l:171", "source": "s:214", "status": "whole" }, { "lowered": "l:172", "source": "s:215", "status": "whole" }, { "lowered": "l:173", "source": "s:216", "status": "whole" }, { "lowered": "l:174", "source": "s:217", "status": "whole" }, { "lowered": "l:175", "source": "s:218", "status": "whole" }, { "lowered": "l:176", "source": "s:219", "status": "whole" }, { "lowered": "l:177", "source": "s:220", "status": "whole" }, { "lowered": "l:178", "source": "s:221", "status": "whole" }, { "source": "s:222", "status": "not_materialized" }, { "lowered": "l:231", "source": "s:223", "status": "whole" }, { "source": "s:224", "status": "not_materialized" }, { "source": "s:225", "status": "not_materialized" }, { "source": "s:226", "status": "not_materialized" }, { "source": "s:227", "status": "not_materialized" }, { "lowered": "l:179", "source": "s:228", "status": "whole" }, { "source": "s:229", "status": "not_materialized" }, { "source": "s:230", "status": "not_materialized" }, { "source": "s:231", "status": "not_materialized" }, { "source": "s:232", "status": "not_materialized" }, { "lowered": "l:180", "source": "s:233", "status": "whole" }, { "source": "s:234", "status": "not_materialized" }, { "source": "s:235", "status": "not_materialized" }, { "source": "s:236", "status": "not_materialized" }, { "source": "s:237", "status": "not_materialized" }, { "source": "s:238", "status": "not_materialized" }, { "lowered": "l:182", "source": "s:239", "status": "whole" }, { "lowered": "l:183", "source": "s:240", "status": "whole" }, { "lowered": "l:184", "source": "s:241", "status": "whole" }, { "lowered": "l:185", "source": "s:242", "status": "whole" }, { "lowered": "l:186", "source": "s:243", "status": "whole" }, { "lowered": "l:187", "source": "s:244", "status": "whole" }, { "lowered": "l:188", "source": "s:245", "status": "whole" }, { "lowered": "l:189", "source": "s:246", "status": "whole" }, { "lowered": "l:190", "source": "s:247", "status": "whole" }, { "lowered": "l:191", "source": "s:248", "status": "whole" }, { "source": "s:249", "status": "not_materialized" }, { "source": "s:250", "status": "not_materialized" }, { "source": "s:251", "status": "not_materialized" }, { "source": "s:252", "status": "not_materialized" }, { "source": "s:253", "status": "not_materialized" }, { "source": "s:254", "status": "not_materialized" }, { "source": "s:255", "status": "not_materialized" }, { "source": "s:256", "status": "not_materialized" }, { "source": "s:257", "status": "not_materialized" }, { "source": "s:258", "status": "not_materialized" }, { "lowered": "l:192", "source": "s:259", "status": "whole" }, { "lowered": "l:193", "source": "s:260", "status": "whole" }, { "lowered": "l:194", "source": "s:261", "status": "whole" }, { "lowered": "l:195", "source": "s:262", "status": "whole" }, { "lowered": "l:196", "source": "s:263", "status": "whole" }, { "lowered": "l:197", "source": "s:264", "status": "whole" }, { "lowered": "l:198", "source": "s:265", "status": "whole" }, { "lowered": "l:199", "source": "s:266", "status": "whole" }, { "lowered": "l:200", "source": "s:267", "status": "whole" }, { "lowered": "l:201", "source": "s:268", "status": "whole" }, { "source": "s:269", "status": "not_materialized" }, { "source": "s:270", "status": "not_materialized" }, { "source": "s:271", "status": "not_materialized" }, { "source": "s:272", "status": "not_materialized" }, { "source": "s:273", "status": "not_materialized" }, { "source": "s:274", "status": "not_materialized" }, { "source": "s:275", "status": "not_materialized" }, { "source": "s:276", "status": "not_materialized" }, { "source": "s:277", "status": "not_materialized" }, { "source": "s:278", "status": "not_materialized" }, { "source": "s:279", "status": "not_materialized" }, { "lowered": "l:202", "source": "s:280", "status": "whole" }, { "lowered": "l:203", "source": "s:281", "status": "whole" }, { "lowered": "l:204", "source": "s:282", "status": "whole" }, { "lowered": "l:205", "source": "s:283", "status": "whole" }, { "lowered": "l:206", "source": "s:284", "status": "whole" }, { "lowered": "l:207", "source": "s:285", "status": "whole" }, { "lowered": "l:208", "source": "s:286", "status": "whole" }, { "source": "s:287", "status": "not_materialized" }, { "lowered": "l:209", "source": "s:288", "status": "whole" }, { "lowered": "l:210", "source": "s:289", "status": "whole" }, { "lowered": "l:211", "source": "s:290", "status": "whole" }, { "source": "s:291", "status": "not_materialized" }, { "source": "s:292", "status": "not_materialized" }, { "source": "s:293", "status": "not_materialized" }, { "lowered": "l:212", "source": "s:294", "status": "whole" }, { "lowered": "l:213", "source": "s:295", "status": "whole" }, { "lowered": "l:214", "source": "s:296", "status": "whole" }, { "lowered": "l:215", "source": "s:297", "status": "whole" }, { "lowered": "l:216", "source": "s:298", "status": "whole" }, { "lowered": "l:217", "source": "s:299", "status": "whole" }, { "lowered": "l:218", "source": "s:300", "status": "whole" }, { "lowered": "l:219", "source": "s:301", "status": "whole" }, { "lowered": "l:220", "source": "s:302", "status": "whole" }, { "lowered": "l:221", "source": "s:303", "status": "whole" }, { "source": "s:304", "status": "not_materialized" }, { "source": "s:305", "status": "not_materialized" }, { "source": "s:306", "status": "not_materialized" }, { "source": "s:307", "status": "not_materialized" }, { "source": "s:308", "status": "not_materialized" }, { "source": "s:309", "status": "not_materialized" }, { "source": "s:310", "status": "not_materialized" }, { "source": "s:311", "status": "not_materialized" }, { "lowered": "l:222", "source": "s:312", "status": "whole" }, { "lowered": "l:223", "source": "s:313", "status": "whole" }, { "lowered": "l:224", "source": "s:314", "status": "whole" }, { "lowered": "l:225", "source": "s:315", "status": "whole" }, { "lowered": "l:226", "source": "s:316", "status": "whole" }, { "lowered": "l:227", "source": "s:317", "status": "whole" }, { "lowered": "l:228", "source": "s:318", "status": "whole" }, { "lowered": "l:229", "source": "s:319", "status": "whole" }, { "source": "s:320", "status": "not_materialized" }, { "source": "s:321", "status": "not_materialized" }, { "source": "s:322", "status": "not_materialized" }, { "source": "s:323", "status": "not_materialized" }, { "source": "s:324", "status": "not_materialized" }, { "source": "s:325", "status": "not_materialized" }, { "source": "s:326", "status": "not_materialized" }, { "source": "s:327", "status": "not_materialized" }, { "source": "s:328", "status": "not_materialized" }, { "source": "s:329", "status": "not_materialized" }, { "source": "s:330", "status": "not_materialized" }, { "source": "s:331", "status": "not_materialized" }, { "lowered": "l:230", "source": "s:332", "status": "whole" }, { "lowered": "l:232", "source": "s:333", "status": "whole" }, { "lowered": "l:233", "source": "s:334", "status": "whole" }, { "lowered": "l:234", "source": "s:335", "status": "whole" }, { "lowered": "l:235", "source": "s:336", "status": "whole" }, { "lowered": "l:236", "source": "s:337", "status": "whole" }, { "lowered": "l:237", "source": "s:338", "status": "whole" }, { "lowered": "l:238", "source": "s:339", "status": "whole" }, { "source": "s:340", "status": "not_materialized" }, { "source": "s:341", "status": "not_materialized" }, { "source": "s:342", "status": "not_materialized" }, { "source": "s:343", "status": "not_materialized" }, { "lowered": "l:239", "source": "s:344", "status": "whole" }, { "source": "s:345", "status": "not_materialized" }, { "source": "s:346", "status": "not_materialized" }, { "source": "s:347", "status": "not_materialized" }, { "lowered": "l:241", "source": "s:348", "status": "whole" }, { "lowered": "l:242", "source": "s:349", "status": "whole" }, { "source": "s:350", "status": "not_materialized" }, { "source": "s:351", "status": "not_materialized" }, { "lowered": "l:243", "source": "s:352", "status": "whole" }, { "lowered": "l:244", "source": "s:353", "status": "whole" }, { "source": "s:354", "status": "not_materialized" }, { "source": "s:355", "status": "not_materialized" }, { "source": "s:356", "status": "not_materialized" }, { "source": "s:357", "status": "not_materialized" }, { "source": "s:358", "status": "not_materialized" }, { "lowered": "l:245", "source": "s:359", "status": "whole" }, { "lowered": "l:246", "source": "s:360", "status": "whole" }, { "lowered": "l:247", "source": "s:361", "status": "whole" }, { "lowered": "l:248", "source": "s:362", "status": "whole" }, { "source": "s:363", "status": "not_materialized" }, { "source": "s:364", "status": "not_materialized" }, { "source": "s:365", "status": "not_materialized" }, { "source": "s:366", "status": "not_materialized" }, { "lowered": "l:249", "source": "s:367", "status": "whole" }, { "source": "s:368", "status": "not_materialized" }, { "source": "s:369", "status": "not_materialized" }, { "source": "s:370", "status": "not_materialized" }, { "lowered": "l:251", "source": "s:371", "status": "whole" }, { "lowered": "l:252", "source": "s:372", "status": "whole" }, { "source": "s:373", "status": "not_materialized" }, { "source": "s:374", "status": "not_materialized" }, { "lowered": "l:253", "source": "s:375", "status": "whole" }, { "lowered": "l:254", "source": "s:376", "status": "whole" }, { "source": "s:377", "status": "not_materialized" }, { "source": "s:378", "status": "not_materialized" }, { "source": "s:379", "status": "not_materialized" }, { "source": "s:380", "status": "not_materialized" }, { "lowered": "l:255", "source": "s:381", "status": "whole" }, { "lowered": "l:256", "source": "s:382", "status": "whole" }, { "source": "s:383", "status": "not_materialized" }, { "source": "s:384", "status": "not_materialized" }, { "source": "s:385", "status": "not_materialized" }, { "lowered": "l:257", "source": "s:386", "status": "whole" }, { "lowered": "l:258", "source": "s:387", "status": "whole" }, { "lowered": "l:259", "source": "s:388", "status": "whole" }, { "lowered": "l:260", "source": "s:389", "status": "whole" }, { "lowered": "l:261", "source": "s:390", "status": "whole" }, { "lowered": "l:262", "source": "s:391", "status": "whole" }, { "source": "s:392", "status": "not_materialized" }, { "source": "s:393", "status": "not_materialized" }, { "source": "s:394", "status": "not_materialized" }, { "source": "s:395", "status": "not_materialized" }, { "lowered": "l:263", "source": "s:396", "status": "whole" }, { "source": "s:397", "status": "not_materialized" }, { "source": "s:398", "status": "not_materialized" }, { "source": "s:399", "status": "not_materialized" }, { "source": "s:400", "status": "not_materialized" }, { "source": "s:401", "status": "not_materialized" }, { "source": "s:402", "status": "not_materialized" }, { "source": "s:403", "status": "not_materialized" }, { "source": "s:404", "status": "not_materialized" }, { "source": "s:405", "status": "not_materialized" }, { "source": "s:406", "status": "not_materialized" }, { "source": "s:407", "status": "not_materialized" }, { "source": "s:408", "status": "not_materialized" }, { "lowered": "l:265", "source": "s:409", "status": "whole" }, { "lowered": "l:266", "source": "s:410", "status": "whole" }, { "lowered": "l:267", "source": "s:411", "status": "whole" }, { "lowered": "l:268", "source": "s:412", "status": "whole" }, { "lowered": "l:269", "source": "s:413", "status": "whole" }, { "source": "s:414", "status": "not_materialized" }, { "source": "s:415", "status": "not_materialized" }, { "lowered": "l:270", "source": "s:416", "status": "whole" }, { "source": "s:417", "status": "not_materialized" }, { "lowered": "l:271", "source": "s:418", "status": "whole" }, { "lowered": "l:273", "source": "s:419", "status": "whole" }, { "source": "s:420", "status": "not_materialized" }, { "lowered": "l:274", "source": "s:421", "status": "whole" }, { "lowered": "l:275", "source": "s:422", "status": "whole" }, { "lowered": "l:276", "source": "s:423", "status": "whole" }, { "lowered": "l:277", "source": "s:424", "status": "whole" }, { "lowered": "l:278", "source": "s:425", "status": "whole" }, { "lowered": "l:279", "source": "s:426", "status": "whole" }, { "lowered": "l:280", "source": "s:427", "status": "whole" }, { "lowered": "l:281", "source": "s:428", "status": "whole" }, { "lowered": "l:282", "source": "s:429", "status": "whole" }, { "lowered": "l:283", "source": "s:430", "status": "whole" }, { "lowered": "l:284", "source": "s:431", "status": "whole" }, { "lowered": "l:285", "source": "s:432", "status": "whole" }, { "lowered": "l:286", "source": "s:433", "status": "whole" }, { "lowered": "l:287", "source": "s:434", "status": "whole" }, { "lowered": "l:288", "source": "s:435", "status": "whole" }, { "lowered": "l:289", "source": "s:436", "status": "whole" }, { "lowered": "l:290", "source": "s:437", "status": "whole" }, { "lowered": "l:291", "source": "s:438", "status": "whole" }, { "source": "s:439", "status": "not_materialized" }, { "source": "s:440", "status": "not_materialized" }, { "lowered": "l:292", "source": "s:441", "status": "whole" }, { "source": "s:442", "status": "not_materialized" }, { "lowered": "l:293", "source": "s:443", "status": "whole" }, { "lowered": "l:294", "source": "s:444", "status": "whole" }, { "source": "s:445", "status": "not_materialized" }, { "lowered": "l:295", "source": "s:446", "status": "whole" }, { "lowered": "l:296", "source": "s:447", "status": "whole" }, { "lowered": "l:297", "source": "s:448", "status": "whole" }, { "lowered": "l:298", "source": "s:449", "status": "whole" }, { "lowered": "l:299", "source": "s:450", "status": "whole" }, { "lowered": "l:300", "source": "s:451", "status": "whole" }, { "lowered": "l:301", "source": "s:452", "status": "whole" }, { "lowered": "l:302", "source": "s:453", "status": "whole" }, { "lowered": "l:303", "source": "s:454", "status": "whole" }, { "lowered": "l:304", "source": "s:455", "status": "whole" }, { "lowered": "l:305", "source": "s:456", "status": "whole" }, { "lowered": "l:306", "source": "s:457", "status": "whole" }, { "lowered": "l:307", "source": "s:458", "status": "whole" }, { "lowered": "l:308", "source": "s:459", "status": "whole" }, { "lowered": "l:309", "source": "s:460", "status": "whole" }, { "lowered": "l:310", "source": "s:461", "status": "whole" }, { "lowered": "l:311", "source": "s:462", "status": "whole" }, { "lowered": "l:312", "source": "s:463", "status": "whole" }, { "lowered": "l:313", "source": "s:464", "status": "whole" }, { "lowered": "l:314", "source": "s:465", "status": "whole" }, { "lowered": "l:315", "source": "s:466", "status": "whole" }, { "lowered": "l:316", "source": "s:467", "status": "whole" }, { "lowered": "l:317", "source": "s:468", "status": "whole" }, { "lowered": "l:318", "source": "s:469", "status": "whole" }, { "lowered": "l:319", "source": "s:470", "status": "whole" }, { "lowered": "l:320", "source": "s:471", "status": "whole" }, { "lowered": "l:321", "source": "s:472", "status": "whole" }, { "lowered": "l:322", "source": "s:473", "status": "whole" }, { "source": "s:474", "status": "not_materialized" }, { "source": "s:475", "status": "not_materialized" }, { "lowered": "l:323", "source": "s:476", "status": "whole" }, { "lowered": "l:324", "source": "s:477", "status": "whole" }, { "lowered": "l:325", "source": "s:478", "status": "whole" }, { "lowered": "l:326", "source": "s:479", "status": "whole" }, { "source": "s:480", "status": "not_materialized" }, { "lowered": "l:327", "source": "s:481", "status": "whole" }, { "lowered": "l:328", "source": "s:482", "status": "whole" }, { "lowered": "l:329", "source": "s:483", "status": "whole" }, { "lowered": "l:330", "source": "s:484", "status": "whole" }, { "lowered": "l:331", "source": "s:485", "status": "whole" }, { "lowered": "l:332", "source": "s:486", "status": "whole" }, { "lowered": "l:333", "source": "s:487", "status": "whole" }, { "lowered": "l:334", "source": "s:488", "status": "whole" }, { "lowered": "l:335", "source": "s:489", "status": "whole" }, { "lowered": "l:336", "source": "s:490", "status": "whole" }, { "lowered": "l:337", "source": "s:491", "status": "whole" }, { "lowered": "l:338", "source": "s:492", "status": "whole" }, { "lowered": "l:339", "source": "s:493", "status": "whole" }, { "lowered": "l:340", "source": "s:494", "status": "whole" }, { "lowered": "l:341", "source": "s:495", "status": "whole" }, { "lowered": "l:342", "source": "s:496", "status": "whole" }, { "lowered": "l:343", "source": "s:497", "status": "whole" }, { "lowered": "l:344", "source": "s:498", "status": "whole" }, { "lowered": "l:345", "source": "s:499", "status": "whole" }, { "lowered": "l:346", "source": "s:500", "status": "whole" }, { "lowered": "l:347", "source": "s:501", "status": "whole" }, { "lowered": "l:348", "source": "s:502", "status": "whole" }, { "lowered": "l:349", "source": "s:503", "status": "whole" }, { "lowered": "l:350", "source": "s:504", "status": "whole" }, { "lowered": "l:351", "source": "s:505", "status": "whole" }, { "lowered": "l:352", "source": "s:506", "status": "whole" }, { "lowered": "l:353", "source": "s:507", "status": "whole" }, { "lowered": "l:354", "source": "s:508", "status": "whole" }, { "lowered": "l:355", "source": "s:509", "status": "whole" }, { "lowered": "l:356", "source": "s:510", "status": "whole" }, { "lowered": "l:357", "source": "s:511", "status": "whole" }, { "lowered": "l:358", "source": "s:512", "status": "whole" }, { "lowered": "l:359", "source": "s:513", "status": "whole" }, { "lowered": "l:360", "source": "s:514", "status": "whole" }, { "lowered": "l:361", "source": "s:515", "status": "whole" }, { "lowered": "l:362", "source": "s:516", "status": "whole" }, { "lowered": "l:363", "source": "s:517", "status": "whole" }, { "lowered": "l:364", "source": "s:518", "status": "whole" }, { "lowered": "l:365", "source": "s:519", "status": "whole" }, { "lowered": "l:366", "source": "s:520", "status": "whole" }, { "lowered": "l:367", "source": "s:521", "status": "whole" }, { "lowered": "l:368", "source": "s:522", "status": "whole" }, { "lowered": "l:369", "source": "s:523", "status": "whole" }, { "lowered": "l:370", "source": "s:524", "status": "whole" }, { "lowered": "l:371", "source": "s:525", "status": "whole" }, { "lowered": "l:372", "source": "s:526", "status": "whole" }, { "lowered": "l:373", "source": "s:527", "status": "whole" }, { "lowered": "l:374", "source": "s:528", "status": "whole" }, { "lowered": "l:375", "source": "s:529", "status": "whole" }, { "lowered": "l:376", "source": "s:530", "status": "whole" }, { "lowered": "l:377", "source": "s:531", "status": "whole" }, { "lowered": "l:378", "source": "s:532", "status": "whole" }, { "lowered": "l:379", "source": "s:533", "status": "whole" }, { "source": "s:534", "status": "not_materialized" }, { "source": "s:535", "status": "not_materialized" }, { "lowered": "l:380", "source": "s:536", "status": "whole" }, { "lowered": "l:381", "source": "s:537", "status": "whole" }, { "source": "s:538", "status": "not_materialized" }, { "source": "s:539", "status": "not_materialized" }, { "source": "s:540", "status": "not_materialized" }, { "lowered": "l:383", "source": "s:541", "status": "whole" }, { "lowered": "l:384", "source": "s:542", "status": "whole" }, { "lowered": "l:385", "source": "s:543", "status": "whole" }, { "lowered": "l:386", "source": "s:544", "status": "whole" }, { "lowered": "l:387", "source": "s:545", "status": "whole" }, { "lowered": "l:388", "source": "s:546", "status": "whole" }, { "lowered": "l:389", "source": "s:547", "status": "whole" }, { "lowered": "l:390", "source": "s:548", "status": "whole" }, { "lowered": "l:391", "source": "s:549", "status": "whole" }, { "lowered": "l:392", "source": "s:550", "status": "whole" }, { "lowered": "l:393", "source": "s:551", "status": "whole" }, { "lowered": "l:394", "source": "s:552", "status": "whole" }, { "lowered": "l:395", "source": "s:553", "status": "whole" }, { "lowered": "l:396", "source": "s:554", "status": "whole" }, { "lowered": "l:397", "source": "s:555", "status": "whole" }, { "lowered": "l:398", "source": "s:556", "status": "whole" }, { "lowered": "l:399", "source": "s:557", "status": "whole" }, { "lowered": "l:400", "source": "s:558", "status": "whole" }, { "lowered": "l:401", "source": "s:559", "status": "whole" }, { "source": "s:560", "status": "not_materialized" }, { "source": "s:561", "status": "not_materialized" }, { "lowered": "l:402", "source": "s:562", "status": "whole" }, { "source": "s:563", "status": "not_materialized" }, { "lowered": "l:403", "source": "s:564", "status": "whole" }, { "lowered": "l:404", "source": "s:565", "status": "whole" }, { "source": "s:566", "status": "not_materialized" }, { "lowered": "l:405", "source": "s:567", "status": "whole" }, { "lowered": "l:406", "source": "s:568", "status": "whole" }, { "lowered": "l:407", "source": "s:569", "status": "whole" }, { "lowered": "l:408", "source": "s:570", "status": "whole" }, { "lowered": "l:409", "source": "s:571", "status": "whole" }, { "lowered": "l:410", "source": "s:572", "status": "whole" }, { "lowered": "l:411", "source": "s:573", "status": "whole" }, { "lowered": "l:412", "source": "s:574", "status": "whole" }, { "lowered": "l:413", "source": "s:575", "status": "whole" }, { "lowered": "l:414", "source": "s:576", "status": "whole" }, { "lowered": "l:415", "source": "s:577", "status": "whole" }, { "lowered": "l:416", "source": "s:578", "status": "whole" }, { "lowered": "l:417", "source": "s:579", "status": "whole" }, { "lowered": "l:418", "source": "s:580", "status": "whole" }, { "lowered": "l:419", "source": "s:581", "status": "whole" }, { "lowered": "l:420", "source": "s:582", "status": "whole" }, { "lowered": "l:421", "source": "s:583", "status": "whole" }, { "lowered": "l:422", "source": "s:584", "status": "whole" }, { "lowered": "l:423", "source": "s:585", "status": "whole" }, { "lowered": "l:424", "source": "s:586", "status": "whole" }, { "lowered": "l:425", "source": "s:587", "status": "whole" }, { "lowered": "l:426", "source": "s:588", "status": "whole" }, { "source": "s:589", "status": "not_materialized" }, { "lowered": "l:427", "source": "s:590", "status": "whole" }, { "lowered": "l:428", "source": "s:591", "status": "whole" }, { "lowered": "l:429", "source": "s:592", "status": "whole" }, { "source": "s:593", "status": "not_materialized" }, { "lowered": "l:430", "source": "s:594", "status": "whole" }, { "lowered": "l:431", "source": "s:595", "status": "whole" }, { "lowered": "l:432", "source": "s:596", "status": "whole" }, { "lowered": "l:433", "source": "s:597", "status": "whole" }, { "lowered": "l:434", "source": "s:598", "status": "whole" }, { "lowered": "l:435", "source": "s:599", "status": "whole" }, { "lowered": "l:436", "source": "s:600", "status": "whole" }, { "lowered": "l:437", "source": "s:601", "status": "whole" }, { "lowered": "l:438", "source": "s:602", "status": "whole" }, { "lowered": "l:439", "source": "s:603", "status": "whole" }, { "lowered": "l:440", "source": "s:604", "status": "whole" }, { "lowered": "l:441", "source": "s:605", "status": "whole" }, { "lowered": "l:442", "source": "s:606", "status": "whole" }, { "lowered": "l:443", "source": "s:607", "status": "whole" }, { "lowered": "l:444", "source": "s:608", "status": "whole" }, { "lowered": "l:445", "source": "s:609", "status": "whole" }, { "lowered": "l:446", "source": "s:610", "status": "whole" }, { "lowered": "l:447", "source": "s:611", "status": "whole" }, { "source": "s:612", "status": "not_materialized" }, { "lowered": "l:448", "source": "s:613", "status": "whole" }, { "lowered": "l:449", "source": "s:614", "status": "whole" }, { "lowered": "l:450", "source": "s:615", "status": "whole" }, { "source": "s:616", "status": "not_materialized" }, { "lowered": "l:451", "source": "s:617", "status": "whole" }, { "lowered": "l:452", "source": "s:618", "status": "whole" }, { "lowered": "l:453", "source": "s:619", "status": "whole" }, { "lowered": "l:454", "source": "s:620", "status": "whole" }, { "lowered": "l:455", "source": "s:621", "status": "whole" }, { "lowered": "l:456", "source": "s:622", "status": "whole" }, { "lowered": "l:457", "source": "s:623", "status": "whole" }, { "lowered": "l:458", "source": "s:624", "status": "whole" }, { "lowered": "l:459", "source": "s:625", "status": "whole" }, { "lowered": "l:460", "source": "s:626", "status": "whole" }, { "lowered": "l:461", "source": "s:627", "status": "whole" }, { "lowered": "l:462", "source": "s:628", "status": "whole" }, { "lowered": "l:463", "source": "s:629", "status": "whole" }, { "lowered": "l:464", "source": "s:630", "status": "whole" }, { "lowered": "l:465", "source": "s:631", "status": "whole" }, { "source": "s:632", "status": "not_materialized" }, { "lowered": "l:466", "source": "s:633", "status": "whole" }, { "lowered": "l:467", "source": "s:634", "status": "whole" }, { "lowered": "l:468", "source": "s:635", "status": "whole" }, { "source": "s:636", "status": "not_materialized" }, { "lowered": "l:469", "source": "s:637", "status": "whole" }, { "lowered": "l:470", "source": "s:638", "status": "whole" }, { "lowered": "l:471", "source": "s:639", "status": "whole" }, { "lowered": "l:472", "source": "s:640", "status": "whole" }, { "lowered": "l:473", "source": "s:641", "status": "whole" }, { "lowered": "l:474", "source": "s:642", "status": "whole" }, { "lowered": "l:475", "source": "s:643", "status": "whole" }, { "source": "s:644", "status": "not_materialized" }, { "lowered": "l:476", "source": "s:645", "status": "whole" }, { "lowered": "l:477", "source": "s:646", "status": "whole" }, { "lowered": "l:478", "source": "s:647", "status": "whole" }, { "source": "s:648", "status": "not_materialized" }, { "lowered": "l:479", "source": "s:649", "status": "whole" }, { "lowered": "l:480", "source": "s:650", "status": "whole" }, { "lowered": "l:481", "source": "s:651", "status": "whole" }, { "lowered": "l:482", "source": "s:652", "status": "whole" }, { "lowered": "l:483", "source": "s:653", "status": "whole" }, { "lowered": "l:484", "source": "s:654", "status": "whole" }, { "lowered": "l:485", "source": "s:655", "status": "whole" }, { "source": "s:656", "status": "not_materialized" }, { "lowered": "l:486", "source": "s:657", "status": "whole" }, { "lowered": "l:487", "source": "s:658", "status": "whole" }, { "lowered": "l:488", "source": "s:659", "status": "whole" }, { "source": "s:660", "status": "not_materialized" }, { "lowered": "l:489", "source": "s:661", "status": "whole" }, { "lowered": "l:490", "source": "s:662", "status": "whole" }, { "lowered": "l:491", "source": "s:663", "status": "whole" }, { "lowered": "l:492", "source": "s:664", "status": "whole" }, { "lowered": "l:493", "source": "s:665", "status": "whole" }, { "lowered": "l:494", "source": "s:666", "status": "whole" }, { "lowered": "l:495", "source": "s:667", "status": "whole" }, { "source": "s:668", "status": "not_materialized" }, { "lowered": "l:496", "source": "s:669", "status": "whole" }, { "lowered": "l:497", "source": "s:670", "status": "whole" }, { "lowered": "l:498", "source": "s:671", "status": "whole" }, { "source": "s:672", "status": "not_materialized" }, { "lowered": "l:499", "source": "s:673", "status": "whole" }, { "lowered": "l:500", "source": "s:674", "status": "whole" }, { "lowered": "l:501", "source": "s:675", "status": "whole" }, { "lowered": "l:502", "source": "s:676", "status": "whole" }, { "lowered": "l:503", "source": "s:677", "status": "whole" }, { "lowered": "l:504", "source": "s:678", "status": "whole" }, { "lowered": "l:505", "source": "s:679", "status": "whole" }, { "source": "s:680", "status": "not_materialized" }, { "lowered": "l:506", "source": "s:681", "status": "whole" }, { "lowered": "l:507", "source": "s:682", "status": "whole" }, { "lowered": "l:508", "source": "s:683", "status": "whole" }, { "source": "s:684", "status": "not_materialized" }, { "lowered": "l:509", "source": "s:685", "status": "whole" }, { "lowered": "l:510", "source": "s:686", "status": "whole" }, { "lowered": "l:511", "source": "s:687", "status": "whole" }, { "lowered": "l:512", "source": "s:688", "status": "whole" }, { "lowered": "l:513", "source": "s:689", "status": "whole" } ], "source": { "nodes": [ { "annotation": null, "id": "s:0", "inputs": [ "s:1" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "3422cbe5c1e8b5b1e016e89ff3387bd4290361fdb17782c50e53c187cf575c5d", "shape": [ 34, 35 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e0ebe36501a3a3bd49b9c2eca7f757e673188e21386e2ef140b112922c14d71b", "shape": [ 34, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7bd2d0249f58c2a4d46659f07872f284b01435b9afd6b0e16d9551cfac29acb6", "shape": [ 34, 35 ] } ], "region": "r:2", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 34 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=146.70925247975623))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 146.70925247975623, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 146.70925247975623, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:1", "inputs": [ "s:2", "s:10", "s:689" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:1", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 146.70925247975623, "upper_unbounded": false } }, "width": 35 }, { "annotation": null, "id": "s:2", "inputs": [ "s:3", "s:6", "s:7" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:3", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.005, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:3", "inputs": [ "s:4", "s:5" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:3", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:4", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:3", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:5", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:3", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:6", "inputs": [ "s:7", "s:9" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:3", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:7", "inputs": [ "s:8" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "3eb8480c9ef49139f499acac7ad953fae7d8c289cb60b001375681e6691ca7d1", "shape": [ 1, 17 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "region": "r:5", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:8", "inputs": [], "name": null, "op": "Embedding", "parameters": [ { "dtype": "float32", "name": "table", "sha256": "6bbd3059883cf8b4699c1da2e25ef31211d490f885fcaac1f73475d14b493c32", "shape": [ 18, 17 ] } ], "region": "r:6", "scheduling_predecessors": [], "semantics": { "embedding_width": 17, "input_name": "embedding_input", "vocab_size": 18 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:9", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:3", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:10", "inputs": [ "s:11" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d1eb096a138388c9ee051d92d5ac6d8e55f4754d7e7650310fcab089dad79925", "shape": [ 136, 17 ] } ], "region": "r:7", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 146.70925247975623, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:11", "inputs": [ "s:12", "s:586", "s:609", "s:629", "s:641", "s:653", "s:665", "s:677" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:7", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 36.67731311993905, "upper_unbounded": false } }, "width": 136 }, { "annotation": null, "id": "s:12", "inputs": [ "s:13" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:9", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=18.247419462656246))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 18.247419462656246, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 18.247419462656246, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:13", "inputs": [ "s:14", "s:26" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:8", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 18.247419462656246, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:14", "inputs": [ "s:15" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:12", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:15", "inputs": [ "s:16" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:13", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:16", "inputs": [ "s:17", "s:24" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:13", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:17", "inputs": [ "s:18" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "25870a3818cab31267591ff755d69e74c8435d6e091e4ff7c8ca9945f2f3285c", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:15", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.9956489921875, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:18", "inputs": [ "s:19" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:17", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.3418431447269631, "lower_unbounded": false, "upper": 9.50850981139363, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:19", "inputs": [ "s:20" ], "name": "reciprocal_0_32", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6c50dd4f13b1539d4f43be532766dbdf7fb95eaa3941166aa039680e2e300a9d", "shape": [ 32 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "6b66e1d03190b21246c3ec82a52fd11b0466bbcf48208c4e3f55fb2eaec6cc20", "shape": [ 32, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "483afcc6ea8b039d16cfd49cb47e300e3ec28b944e85daf9d068c4c79a838894", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "60ace6d30c5997f9dc18846fc3aac46f4e0ed2220c14b4c15fcb4f3b0cb7a036", "shape": [ 32, 1 ] } ], "region": "r:20", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 32 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.6581568552730369, hi=10.50850981139363))" } ], "claimed_range": { "lower": 0.6581568552730369, "lower_unbounded": false, "upper": 10.50850981139363, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.6581568552730369, "lower_unbounded": false, "upper": 10.50850981139363, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:20", "inputs": [ "s:21", "s:2", "s:22" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "5138effe2b38a13d44fc24c489380fdb2cb5c5b4b28dacceb3e715e35be86483", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "2e9dafc0fe03a4248692ed626f839acc963a37bac252e72fb9b8a38c20074d31", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:21", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "values in [-0.002175503906249987, 1.0]" } ], "claimed_range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:21", "inputs": [], "name": "mean_where_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:21", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:22", "inputs": [ "s:23" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "region": "r:23", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:23", "inputs": [ "s:7" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:24", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:24", "inputs": [ "s:25" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "25870a3818cab31267591ff755d69e74c8435d6e091e4ff7c8ca9945f2f3285c", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:26", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:25", "inputs": [ "s:18" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:27", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -9.50850981139363, "lower_unbounded": false, "upper": 0.3418431447269631, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:26", "inputs": [ "s:27" ], "name": "remove_leading_0s_sum", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d7c028b7eda356619d577d7e3dfef5da2267e2c58f989f7adcb5316c4793dd4c", "shape": [ 204, 17 ] } ], "region": "r:28", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 18.247419462656246, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:27", "inputs": [ "s:28" ], "name": "broadcast_select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2c93ccfa935dc7b48ae36ca9b98d2c3e3e50962957d9f3b464865f5c129d7072", "shape": [ 204, 216 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "08720f852c879614691b2c33f2980888e7cdf29871f0a4e9e60e7446758bfc2e", "shape": [ 204, 204 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7e26b5245da34ef2eb9e8e57159215ac9a011c4de6a4aa42cdbfb51763b7de8f", "shape": [ 204, 216 ] } ], "region": "r:30", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 204 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0149999999999997))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "s:28", "inputs": [ "s:29", "s:567" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:29", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 216 }, { "annotation": null, "id": "s:29", "inputs": [ "s:30" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "ff9beaa3dcaa6fe5b2e7cdb26ebff8cfb3d90db02c236911299cdeedf80443f3", "shape": [ 48 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "cc8cf337ee5ca4135fb2b5843d8c2ecb6b7e65ca5fccad03ef3f84da786c212a", "shape": [ 48, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "81681b33ced1a8f08f6888a042ed160ade67bd948c9a6a0f45f325c25249eec2", "shape": [ 12 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "262fb0b41a348fcbf610dbe142f93f4eae30967bf284fa2438e7550aeaeb59ad", "shape": [ 48, 12 ] } ], "region": "r:32", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 48 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 12 }, { "annotation": null, "id": "s:30", "inputs": [ "s:31", "s:566" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:31", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.023930542968749857, "lower_unbounded": false, "upper": 12.023930542968749, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:31", "inputs": [ "s:32", "s:559" ], "name": null, "op": "Add", "parameters": [], "region": "r:33", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.023930542968749857, "lower_unbounded": false, "upper": 11.023930542968749, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:32", "inputs": [ "s:33" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f603f94b1d517d5138cdeae0c125779455d596b548a58e5cb4d941f97a1a242a", "shape": [ 5, 1 ] } ], "region": "r:34", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.010877519531249935, "lower_unbounded": false, "upper": 5.010877519531249, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:33", "inputs": [ "s:34", "s:414", "s:439", "s:474", "s:534" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:34", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 5 }, { "annotation": null, "id": "s:34", "inputs": [ "s:35" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "region": "r:36", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:35", "inputs": [ "s:36" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:37", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0043510078124998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:36", "inputs": [ "s:37" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "28c4f7df3425ac5087c2632d76786a4afaaf06906eeafb89d8d36ba650e3f24b", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:39", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:37", "inputs": [ "s:38" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:40", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:38", "inputs": [ "s:39" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:40", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:39", "inputs": [ "s:40" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "region": "r:42", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:40", "inputs": [ "s:41" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:43", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:41", "inputs": [ "s:42" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d02095ef9169de77f0d4bcaefe3494a01a6cb5303d8f16874d410303fff7354a", "shape": [ 1, 17 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "region": "r:45", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:42", "inputs": [ "s:43" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9de7dcac2491cd60a9a20fa1b13cbaa6a2ae2547b251a17e2001b747a444d46d", "shape": [ 51, 17 ] } ], "region": "r:47", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:43", "inputs": [ "s:44", "s:72", "s:81" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:47", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 51 }, { "annotation": null, "id": "s:44", "inputs": [ "s:45" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:49", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:45", "inputs": [ "s:46", "s:71" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:48", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:46", "inputs": [ "s:47", "s:50", "s:58" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:50", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.005, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:47", "inputs": [ "s:48", "s:49" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:50", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:48", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:50", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:49", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:50", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:50", "inputs": [ "s:51", "s:70" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:50", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:51", "inputs": [ "s:52" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:53", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:52", "inputs": [ "s:53" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:54", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:53", "inputs": [ "s:54", "s:63" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:54", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:54", "inputs": [ "s:55" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "28c4f7df3425ac5087c2632d76786a4afaaf06906eeafb89d8d36ba650e3f24b", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:57", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:55", "inputs": [ "s:56" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "8a31a40ecac0ceb4d87b30bd156ca7a547e8e33dc071454b765fbc777d1c34a1", "shape": [ 3, 1 ] } ], "region": "r:58", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.00652651171875, "lower_unbounded": false, "upper": 3.00652651171875, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:56", "inputs": [ "s:57", "s:59", "s:61" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:58", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:57", "inputs": [ "s:58" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "aced851223ade84128237ff20c2acca7ac1102b3d2dfea9f7fcfb3eef3457a01", "shape": [ 2, 1 ] } ], "region": "r:60", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.00217550390625, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:58", "inputs": [ "s:8" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "ef7b1f9e5a2d907c7ff2496a32dc2d4879721c404cc224a31671940ba846c4e2", "shape": [ 1, 17 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "region": "r:62", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:59", "inputs": [ "s:60" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "aced851223ade84128237ff20c2acca7ac1102b3d2dfea9f7fcfb3eef3457a01", "shape": [ 2, 1 ] } ], "region": "r:64", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.00217550390625, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:60", "inputs": [ "s:8" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "82c5b2a5f705dd388e12fac162246ad059cfa99089bdd1df5103928849793813", "shape": [ 1, 17 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "region": "r:66", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:61", "inputs": [ "s:62" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "aced851223ade84128237ff20c2acca7ac1102b3d2dfea9f7fcfb3eef3457a01", "shape": [ 2, 1 ] } ], "region": "r:68", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.00217550390625, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:62", "inputs": [ "s:8" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2e5d5c3fc5f63a23d29988ab3982c8da9a63a8cd2639d8ca27ab5390529327d9", "shape": [ 1, 17 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "region": "r:70", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:63", "inputs": [ "s:64" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "f41491c367f49e7206c45724651464310fcb7d17e3059779d1c95aa901f3fce1", "shape": [ 2, 1 ] } ], "region": "r:73", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:64", "inputs": [ "s:65", "s:68", "s:7" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:74", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:65", "inputs": [ "s:66", "s:67" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:74", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:66", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:74", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:67", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:74", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:68", "inputs": [ "s:7", "s:69" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:74", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:69", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:74", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:70", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:50", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:71", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ], "values": [ 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "region": "r:75", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:72", "inputs": [ "s:73" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:77", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:73", "inputs": [ "s:74", "s:80" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:76", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:74", "inputs": [ "s:75", "s:78", "s:60" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:78", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.005, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:75", "inputs": [ "s:76", "s:77" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:78", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:76", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:78", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:77", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:78", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:78", "inputs": [ "s:51", "s:79" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:78", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:79", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:78", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:80", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ], "values": [ 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "region": "r:79", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:81", "inputs": [ "s:82" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:81", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:82", "inputs": [ "s:83", "s:89" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:80", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:83", "inputs": [ "s:84", "s:87", "s:62" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:82", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.005, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:84", "inputs": [ "s:85", "s:86" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:82", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:85", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:82", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:86", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:82", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:87", "inputs": [ "s:51", "s:88" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:82", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:88", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:82", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:89", "inputs": [ "s:90" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "ed95422d29540bfc7f36e115c60ffa7a8494973d7cfe886d9611f90169997399", "shape": [ 200, 17 ] } ], "region": "r:84", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:90", "inputs": [ "s:91", "s:228", "s:280" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:85", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 36 }, { "annotation": null, "id": "s:91", "inputs": [ "s:92" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f29de6d677eb31146fcabd1fd447b4654829f0b547c221f0516d422ce631c077", "shape": [ 100, 17 ] } ], "region": "r:86", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:92", "inputs": [ "s:93" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "7a12e561363385e9dfeeab326368731c030ed4b374e7f5897ac819159d2884c5", "shape": [ 100 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c79c7241ec19383f6be1709969fd023c0c6afaedf0eb11cb5b8fd6e8b99856d5", "shape": [ 100, 100 ] } ], "region": "r:88", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 100 }, { "annotation": null, "id": "s:93", "inputs": [ "s:94" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e985c32fd28bde405da64bc903c85250a82a0fa3d2291eca21abbd52b26f503d", "shape": [ 100 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "926cd87c12dbf5fdf0b1c90cf22cbad6b1d3163cec56c0977117efdb2c3efbd3", "shape": [ 100, 100 ] } ], "region": "r:89", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 100 }, { "annotation": null, "id": "s:94", "inputs": [ "s:95" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "5043c9e1d4cb1fea053e49e1d7a8e09157add36a5a1f3d8ee90efd200a572900", "shape": [ 400 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2872925994dc7f63708f8a149b44e2bd8c152caca1b07d0d22d1db16fdfa7b9e", "shape": [ 400, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "79fd15545a108bf1260d68e92162b968655b2a63fc31cc35e91d084b4cdb7c95", "shape": [ 100 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "322932091bb64b559083df41f6380723142ffa85606836596f3381bffb456144", "shape": [ 400, 100 ] } ], "region": "r:91", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 400 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 100 }, { "annotation": null, "id": "s:95", "inputs": [ "s:96", "s:227" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:90", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 37.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:96", "inputs": [ "s:97" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f603f94b1d517d5138cdeae0c125779455d596b548a58e5cb4d941f97a1a242a", "shape": [ 5, 1 ] } ], "region": "r:92", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 36.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:97", "inputs": [ "s:98", "s:222", "s:224", "s:225", "s:226" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:92", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 36.0, "upper_unbounded": false } }, "width": 5 }, { "annotation": null, "id": "s:98", "inputs": [ "s:99" ], "name": "digit_value", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "23b4c414e1da04e6b50944ee108cf56058564d3fa7601ba5b482a051866afd93", "shape": [ 17, 1 ] } ], "region": null, "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 36.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:99", "inputs": [ "s:100" ], "name": "pp_tens", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "b147e672e8eb1f6dd33bc1430113f435acdd9ad22436ba20e281e1360d8054c9", "shape": [ 34, 17 ] } ], "region": "r:93", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:100", "inputs": [ "s:101" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "a6b06fbd738b910030a0847ec7779449c3c076d0975def1ce5d4755431bb9ade", "shape": [ 34 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "bc9b478bb7986ee8a1778fc116c22bb23e7e7352444e62a46c7953e07a2b57bd", "shape": [ 100, 34 ] } ], "region": "r:95", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:101", "inputs": [ "s:102", "s:166" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:96", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:102", "inputs": [ "s:103", "s:106", "s:108" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] } ], "region": "r:97", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 17, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:103", "inputs": [ "s:104", "s:105" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:97", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:104", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:97", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:105", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:97", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:106", "inputs": [ "s:7", "s:107" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:97", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:107", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:97", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:108", "inputs": [ "s:109", "s:138", "s:163" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "ff1247d951578e37ef4f20b3e4a593814438971e903430bf33e180aaeed8c538", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "54e4e0a85bc6a3e7d002e3c35b74ff13c2449639e18520e333933b32d1fb004a", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] } ], "region": "r:99", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 17, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:109", "inputs": [ "s:110", "s:136" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:100", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.0130530234375, "lower_unbounded": false, "upper": 1.0130530234375, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:110", "inputs": [ "s:111" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "56e475e08e4d688e9ca9d3faa1c854f47fc5b6382bdec3ccc4e64114e57ad6e9", "shape": [ 3, 3 ] } ], "region": "r:102", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:111", "inputs": [ "s:112" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "8a31a40ecac0ceb4d87b30bd156ca7a547e8e33dc071454b765fbc777d1c34a1", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "64eec129c8d3f58ee6b7ca145b25e312fa82d3d276db5adaedb59aaebb824885", "shape": [ 3, 3 ] } ], "region": "r:103", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:112", "inputs": [ "s:113" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "10f5b23f39f81bf5063c468e6bb37dbb7cd44d7381dbc28b54c11a333e817e82", "shape": [ 12 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "1ad43993d87c26d5c610bb4df0a3f9b1b03186bb9c69b26284c08db4a57dfa2e", "shape": [ 12, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3ee6949226ed15c57a63c066edaaac5d2a2988e2ba82c56d47f3d1b7f8620845", "shape": [ 12, 3 ] } ], "region": "r:105", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 12 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:113", "inputs": [ "s:114", "s:135" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:104", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -4.3366749232420485, "lower_unbounded": false, "upper": 1.5033415899087146, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:114", "inputs": [ "s:115" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "71426d210d52fa91812d0a39251aa75ded92519c3d746b8ced4e5a02ec97960d", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:106", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -4.3366749232420485, "lower_unbounded": false, "upper": 0.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:115", "inputs": [ "s:116" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:108", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.3366749232420483, "lower_unbounded": false, "upper": 3.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:116", "inputs": [ "s:117", "s:120", "s:122" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:109", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.3366749232420483, "lower_unbounded": false, "upper": 4.503341589908715, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:117", "inputs": [ "s:118", "s:119" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:109", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:118", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:109", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:119", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:109", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:120", "inputs": [ "s:51", "s:121" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:109", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:121", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:109", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:122", "inputs": [ "s:123" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:112", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.3366749232420483, "lower_unbounded": false, "upper": 4.503341589908715, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:123", "inputs": [ "s:124" ], "name": "reciprocal_0_32", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "86ace85102b9d8a1c0886e9cf2cbb181c89f8ca60093d8ccbe6cbd12a754ad1b", "shape": [ 32 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "5870a33a3d94db2db2aaa92ac9edb125e7803a503405c199042a2f0c6d461a6b", "shape": [ 32, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "41d043a7c0f00b8b39f1365727f09ede61e4b865694b6dc1d4acbbd97029d11b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "a4d06da2357859d57b942afb3157f9957beb9485b57a8ae112cd6e8f5e6b3764", "shape": [ 32, 1 ] } ], "region": "r:115", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 32 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.6633250767579517, hi=5.503341589908715))" } ], "claimed_range": { "lower": 0.6633250767579517, "lower_unbounded": false, "upper": 5.503341589908715, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.6633250767579517, "lower_unbounded": false, "upper": 5.503341589908715, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:124", "inputs": [ "s:125", "s:126", "s:133" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "5138effe2b38a13d44fc24c489380fdb2cb5c5b4b28dacceb3e715e35be86483", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "2e9dafc0fe03a4248692ed626f839acc963a37bac252e72fb9b8a38c20074d31", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:116", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "values in [-0.002175503906249987, 1.0]" } ], "claimed_range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:125", "inputs": [], "name": "mean_where_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:116", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:126", "inputs": [ "s:127", "s:130", "s:131" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:117", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:127", "inputs": [ "s:128", "s:129" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:117", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:128", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:117", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:129", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:117", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:130", "inputs": [ "s:131", "s:132" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:117", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:131", "inputs": [ "s:8" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "e79f394c9a536f65c56b29d1e593771c6e332b653623054cfde6b4c6056348bf", "shape": [ 1, 17 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "region": "r:119", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:132", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:117", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:133", "inputs": [ "s:134" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "region": "r:121", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:134", "inputs": [ "s:131" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:122", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:135", "inputs": [ "s:114" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:123", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.3366749232420485, "lower_unbounded": false, "upper": 1.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:136", "inputs": [ "s:137" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:124", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.0130530234375, "lower_unbounded": false, "upper": 1.0130530234375, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:137", "inputs": [ "s:110" ], "name": "region_no_lane", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3, 1 ] } ], "region": "r:98", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.0130530234375, "lower_unbounded": false, "upper": 0.013053023437499922, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:138", "inputs": [ "s:139", "s:160" ], "name": null, "op": "Add", "parameters": [], "region": "r:125", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004372762851562474, "lower_unbounded": false, "upper": 1.5118727628515622, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:139", "inputs": [ "s:140", "s:159" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:126", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004372762851562474, "lower_unbounded": false, "upper": 1.0093727628515623, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:140", "inputs": [ "s:141" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "acc2e4c62e9b92c67550621c1d3c42498eab8b87200286819aae131530c4dc41", "shape": [ 3, 4 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e5cd6f63a143d03bf475cd19b883bbdecec4efb62d8303f209b0dde6ae0565cd", "shape": [ 3, 3 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "ba88de7e97f50c001a9d0c4e5c690cd5e4d90c7e5a7b64b96b485a3847228885", "shape": [ 3, 4 ] } ], "region": "r:128", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 3 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.004351007812499974, hi=1.0043510078125))" } ], "claimed_range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:141", "inputs": [ "s:142", "s:153" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:127", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:142", "inputs": [ "s:143" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:131", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:143", "inputs": [ "s:144" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:132", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.0478611640624997, "lower_unbounded": false, "upper": 20.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:144", "inputs": [ "s:145", "s:146" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:132", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.04351015625, "lower_unbounded": false, "upper": 19.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:145", "inputs": [ "s:8" ], "name": "map_to_table", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "2c34ce1df23b838c5abf2a7f6437cca3d3067ed509ff25f11df6b11b582b51eb", "shape": [ 10 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "eee183eff6e72006bb7ace8c5ae3f42029abaa0004524c553789b6cca053a22c", "shape": [ 10, 17 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e7a2f4b7a886aabf181b94c496dabd00bd3f8da32a01f3da03e6d6868564aed6", "shape": [ 10, 1 ] } ], "region": "r:135", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 10 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-21.0, hi=19.0))" } ], "claimed_range": { "lower": -21.0, "lower_unbounded": false, "upper": 19.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.04351015625, "lower_unbounded": false, "upper": 19.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:146", "inputs": [ "s:147" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "f41491c367f49e7206c45724651464310fcb7d17e3059779d1c95aa901f3fce1", "shape": [ 2, 1 ] } ], "region": "r:138", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:147", "inputs": [ "s:148", "s:151", "s:51" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:139", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:148", "inputs": [ "s:149", "s:150" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:139", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:149", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:139", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:150", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:139", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:151", "inputs": [ "s:51", "s:152" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:139", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:152", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:139", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:153", "inputs": [ "s:154" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "56e475e08e4d688e9ca9d3faa1c854f47fc5b6382bdec3ccc4e64114e57ad6e9", "shape": [ 3, 3 ] } ], "region": "r:141", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:154", "inputs": [ "s:155" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "8a31a40ecac0ceb4d87b30bd156ca7a547e8e33dc071454b765fbc777d1c34a1", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "64eec129c8d3f58ee6b7ca145b25e312fa82d3d276db5adaedb59aaebb824885", "shape": [ 3, 3 ] } ], "region": "r:142", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:155", "inputs": [ "s:156" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "10f5b23f39f81bf5063c468e6bb37dbb7cd44d7381dbc28b54c11a333e817e82", "shape": [ 12 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "1ad43993d87c26d5c610bb4df0a3f9b1b03186bb9c69b26284c08db4a57dfa2e", "shape": [ 12, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3ee6949226ed15c57a63c066edaaac5d2a2988e2ba82c56d47f3d1b7f8620845", "shape": [ 12, 3 ] } ], "region": "r:144", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 12 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:156", "inputs": [ "s:157", "s:158" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:143", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.3366749232420483, "lower_unbounded": false, "upper": 4.503341589908715, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:157", "inputs": [ "s:122" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:145", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.3366749232420483, "lower_unbounded": false, "upper": 3.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:158", "inputs": [ "s:157" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:146", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.3366749232420483, "lower_unbounded": false, "upper": 4.503341589908715, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:159", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ], "values": [ 0.0 ] } ], "region": "r:147", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:160", "inputs": [ "s:161" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb", "shape": [ 4 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "6ad01752d1a3928862bafa2e6bbe6b2df45d614b086e065fdacf50cb7180f6f3", "shape": [ 4, 5 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb", "shape": [ 4 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "c91315e209cc3cd1c12fae1264b1597c61d8ddc4fb51cc0941c96fd1be2076b1", "shape": [ 4, 4 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb", "shape": [ 4 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "6e92b4c838f9ab461fc45d72308d5a5360af4497fff417bae914541608979a02", "shape": [ 4, 5 ] } ], "region": "r:149", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 4 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:161", "inputs": [ "s:131", "s:162" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:148", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 5 }, { "annotation": null, "id": "s:162", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "024712320e6f0d6455efd690279f94b44c2b7ac0f5351f626965347a9d3d4199", "shape": [ 4 ], "values": [ 0.5, 0.5, 0.5, 1.0 ] } ], "region": "r:150", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.5, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:163", "inputs": [ "s:164" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "3422cbe5c1e8b5b1e016e89ff3387bd4290361fdb17782c50e53c187cf575c5d", "shape": [ 34, 35 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e0ebe36501a3a3bd49b9c2eca7f757e673188e21386e2ef140b112922c14d71b", "shape": [ 34, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7bd2d0249f58c2a4d46659f07872f284b01435b9afd6b0e16d9551cfac29acb6", "shape": [ 34, 35 ] } ], "region": "r:152", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 34 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:164", "inputs": [ "s:142", "s:8", "s:165" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:151", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 35 }, { "annotation": null, "id": "s:165", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ], "values": [ 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "region": "r:153", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:166", "inputs": [ "s:167", "s:170", "s:172" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] } ], "region": "r:154", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 17, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:167", "inputs": [ "s:168", "s:169" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:154", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:168", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:154", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:169", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:154", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:170", "inputs": [ "s:7", "s:171" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:154", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:171", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:154", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:172", "inputs": [ "s:173", "s:201", "s:219" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "ff1247d951578e37ef4f20b3e4a593814438971e903430bf33e180aaeed8c538", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "54e4e0a85bc6a3e7d002e3c35b74ff13c2449639e18520e333933b32d1fb004a", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] } ], "region": "r:156", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 17, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:173", "inputs": [ "s:174", "s:199" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:157", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.0130530234375, "lower_unbounded": false, "upper": 1.0130530234375, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:174", "inputs": [ "s:175" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "56e475e08e4d688e9ca9d3faa1c854f47fc5b6382bdec3ccc4e64114e57ad6e9", "shape": [ 3, 3 ] } ], "region": "r:159", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:175", "inputs": [ "s:176" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "8a31a40ecac0ceb4d87b30bd156ca7a547e8e33dc071454b765fbc777d1c34a1", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "64eec129c8d3f58ee6b7ca145b25e312fa82d3d276db5adaedb59aaebb824885", "shape": [ 3, 3 ] } ], "region": "r:160", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:176", "inputs": [ "s:177" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "10f5b23f39f81bf5063c468e6bb37dbb7cd44d7381dbc28b54c11a333e817e82", "shape": [ 12 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "1ad43993d87c26d5c610bb4df0a3f9b1b03186bb9c69b26284c08db4a57dfa2e", "shape": [ 12, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3ee6949226ed15c57a63c066edaaac5d2a2988e2ba82c56d47f3d1b7f8620845", "shape": [ 12, 3 ] } ], "region": "r:162", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 12 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:177", "inputs": [ "s:178", "s:198" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:161", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -4.3366749232420485, "lower_unbounded": false, "upper": 1.5033415899087146, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:178", "inputs": [ "s:179" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "71426d210d52fa91812d0a39251aa75ded92519c3d746b8ced4e5a02ec97960d", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:163", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -4.3366749232420485, "lower_unbounded": false, "upper": 0.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:179", "inputs": [ "s:180" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:165", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.3366749232420483, "lower_unbounded": false, "upper": 3.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:180", "inputs": [ "s:181", "s:184", "s:186" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:166", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.3366749232420483, "lower_unbounded": false, "upper": 4.503341589908715, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:181", "inputs": [ "s:182", "s:183" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:166", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:182", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:166", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:183", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:166", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:184", "inputs": [ "s:7", "s:185" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:166", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:185", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:166", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:186", "inputs": [ "s:187" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:169", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.3366749232420483, "lower_unbounded": false, "upper": 4.503341589908715, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:187", "inputs": [ "s:188" ], "name": "reciprocal_0_32", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "86ace85102b9d8a1c0886e9cf2cbb181c89f8ca60093d8ccbe6cbd12a754ad1b", "shape": [ 32 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "5870a33a3d94db2db2aaa92ac9edb125e7803a503405c199042a2f0c6d461a6b", "shape": [ 32, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "41d043a7c0f00b8b39f1365727f09ede61e4b865694b6dc1d4acbbd97029d11b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "a4d06da2357859d57b942afb3157f9957beb9485b57a8ae112cd6e8f5e6b3764", "shape": [ 32, 1 ] } ], "region": "r:172", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 32 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.6633250767579517, hi=5.503341589908715))" } ], "claimed_range": { "lower": 0.6633250767579517, "lower_unbounded": false, "upper": 5.503341589908715, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.6633250767579517, "lower_unbounded": false, "upper": 5.503341589908715, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:188", "inputs": [ "s:189", "s:190", "s:196" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "5138effe2b38a13d44fc24c489380fdb2cb5c5b4b28dacceb3e715e35be86483", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "2e9dafc0fe03a4248692ed626f839acc963a37bac252e72fb9b8a38c20074d31", "shape": [ 1, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:173", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "values in [-0.002175503906249987, 1.0021755039062499]" } ], "claimed_range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:189", "inputs": [], "name": "mean_where_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:173", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:190", "inputs": [ "s:191", "s:194", "s:51" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:174", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 1, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:191", "inputs": [ "s:192", "s:193" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:174", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:192", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:174", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:193", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:174", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:194", "inputs": [ "s:51", "s:195" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:174", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:195", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:174", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:196", "inputs": [ "s:197" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "region": "r:176", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:197", "inputs": [ "s:51" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:177", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0043510078124998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:198", "inputs": [ "s:178" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:178", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.3366749232420485, "lower_unbounded": false, "upper": 1.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:199", "inputs": [ "s:200" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:179", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.0130530234375, "lower_unbounded": false, "upper": 1.0130530234375, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:200", "inputs": [ "s:174" ], "name": "region_no_lane", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3, 1 ] } ], "region": "r:155", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.0130530234375, "lower_unbounded": false, "upper": 0.013053023437499922, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:201", "inputs": [ "s:202", "s:216" ], "name": null, "op": "Add", "parameters": [], "region": "r:180", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004372762851562474, "lower_unbounded": false, "upper": 1.5118727628515622, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:202", "inputs": [ "s:203", "s:215" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:181", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004372762851562474, "lower_unbounded": false, "upper": 1.0093727628515623, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:203", "inputs": [ "s:204" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "acc2e4c62e9b92c67550621c1d3c42498eab8b87200286819aae131530c4dc41", "shape": [ 3, 4 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e5cd6f63a143d03bf475cd19b883bbdecec4efb62d8303f209b0dde6ae0565cd", "shape": [ 3, 3 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "ba88de7e97f50c001a9d0c4e5c690cd5e4d90c7e5a7b64b96b485a3847228885", "shape": [ 3, 4 ] } ], "region": "r:183", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 3 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.004351007812499974, hi=1.0043510078125))" } ], "claimed_range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:204", "inputs": [ "s:205", "s:209" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:182", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:205", "inputs": [ "s:206" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "f22239942c4ac043eb6887cd996054943f2d24a5e56217c21186a366ca97c23d", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:186", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:206", "inputs": [ "s:207" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "8a31a40ecac0ceb4d87b30bd156ca7a547e8e33dc071454b765fbc777d1c34a1", "shape": [ 3, 1 ] } ], "region": "r:187", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.0522121718749995, "lower_unbounded": false, "upper": 21.008702015624998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:207", "inputs": [ "s:145", "s:147", "s:208" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:187", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.04351015625, "lower_unbounded": false, "upper": 19.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:208", "inputs": [ "s:64" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "f41491c367f49e7206c45724651464310fcb7d17e3059779d1c95aa901f3fce1", "shape": [ 2, 1 ] } ], "region": "r:190", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:209", "inputs": [ "s:210" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "56e475e08e4d688e9ca9d3faa1c854f47fc5b6382bdec3ccc4e64114e57ad6e9", "shape": [ 3, 3 ] } ], "region": "r:192", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:210", "inputs": [ "s:211" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "8a31a40ecac0ceb4d87b30bd156ca7a547e8e33dc071454b765fbc777d1c34a1", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "64eec129c8d3f58ee6b7ca145b25e312fa82d3d276db5adaedb59aaebb824885", "shape": [ 3, 3 ] } ], "region": "r:193", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:211", "inputs": [ "s:212" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "10f5b23f39f81bf5063c468e6bb37dbb7cd44d7381dbc28b54c11a333e817e82", "shape": [ 12 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "1ad43993d87c26d5c610bb4df0a3f9b1b03186bb9c69b26284c08db4a57dfa2e", "shape": [ 12, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3ee6949226ed15c57a63c066edaaac5d2a2988e2ba82c56d47f3d1b7f8620845", "shape": [ 12, 3 ] } ], "region": "r:195", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 12 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:212", "inputs": [ "s:213", "s:214" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:194", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.3366749232420483, "lower_unbounded": false, "upper": 4.503341589908715, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:213", "inputs": [ "s:186" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:196", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.3366749232420483, "lower_unbounded": false, "upper": 3.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:214", "inputs": [ "s:213" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:197", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.3366749232420483, "lower_unbounded": false, "upper": 4.503341589908715, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:215", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ], "values": [ 0.0 ] } ], "region": "r:198", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:216", "inputs": [ "s:217" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb", "shape": [ 4 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "6ad01752d1a3928862bafa2e6bbe6b2df45d614b086e065fdacf50cb7180f6f3", "shape": [ 4, 5 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb", "shape": [ 4 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "c91315e209cc3cd1c12fae1264b1597c61d8ddc4fb51cc0941c96fd1be2076b1", "shape": [ 4, 4 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb", "shape": [ 4 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "6e92b4c838f9ab461fc45d72308d5a5360af4497fff417bae914541608979a02", "shape": [ 4, 5 ] } ], "region": "r:200", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 4 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:217", "inputs": [ "s:51", "s:218" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:199", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 5 }, { "annotation": null, "id": "s:218", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "024712320e6f0d6455efd690279f94b44c2b7ac0f5351f626965347a9d3d4199", "shape": [ 4 ], "values": [ 0.5, 0.5, 0.5, 1.0 ] } ], "region": "r:201", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.5, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:219", "inputs": [ "s:220" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "3422cbe5c1e8b5b1e016e89ff3387bd4290361fdb17782c50e53c187cf575c5d", "shape": [ 34, 35 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e0ebe36501a3a3bd49b9c2eca7f757e673188e21386e2ef140b112922c14d71b", "shape": [ 34, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7bd2d0249f58c2a4d46659f07872f284b01435b9afd6b0e16d9551cfac29acb6", "shape": [ 34, 35 ] } ], "region": "r:203", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 34 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:220", "inputs": [ "s:205", "s:8", "s:221" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:202", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 35 }, { "annotation": null, "id": "s:221", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ], "values": [ 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "region": "r:204", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:222", "inputs": [ "s:223" ], "name": "digit_value", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "23b4c414e1da04e6b50944ee108cf56058564d3fa7601ba5b482a051866afd93", "shape": [ 17, 1 ] } ], "region": null, "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:223", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ], "values": [ 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "region": "r:205", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:224", "inputs": [ "s:223" ], "name": "digit_value", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "23b4c414e1da04e6b50944ee108cf56058564d3fa7601ba5b482a051866afd93", "shape": [ 17, 1 ] } ], "region": null, "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:225", "inputs": [ "s:223" ], "name": "digit_value", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "23b4c414e1da04e6b50944ee108cf56058564d3fa7601ba5b482a051866afd93", "shape": [ 17, 1 ] } ], "region": null, "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:226", "inputs": [ "s:223" ], "name": "digit_value", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "23b4c414e1da04e6b50944ee108cf56058564d3fa7601ba5b482a051866afd93", "shape": [ 17, 1 ] } ], "region": null, "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:227", "inputs": [ "s:96" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:206", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 37.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:228", "inputs": [ "s:229" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "6f8067031a2f2304a92ac0120dc3bace57a8632405b8843002c93959dfbaab6f", "shape": [ 100, 17 ] } ], "region": "r:207", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:229", "inputs": [ "s:230" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "7a12e561363385e9dfeeab326368731c030ed4b374e7f5897ac819159d2884c5", "shape": [ 100 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c79c7241ec19383f6be1709969fd023c0c6afaedf0eb11cb5b8fd6e8b99856d5", "shape": [ 100, 100 ] } ], "region": "r:209", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 100 }, { "annotation": null, "id": "s:230", "inputs": [ "s:231" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e985c32fd28bde405da64bc903c85250a82a0fa3d2291eca21abbd52b26f503d", "shape": [ 100 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "926cd87c12dbf5fdf0b1c90cf22cbad6b1d3163cec56c0977117efdb2c3efbd3", "shape": [ 100, 100 ] } ], "region": "r:210", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 100 }, { "annotation": null, "id": "s:231", "inputs": [ "s:232" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "5043c9e1d4cb1fea053e49e1d7a8e09157add36a5a1f3d8ee90efd200a572900", "shape": [ 400 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2872925994dc7f63708f8a149b44e2bd8c152caca1b07d0d22d1db16fdfa7b9e", "shape": [ 400, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "79fd15545a108bf1260d68e92162b968655b2a63fc31cc35e91d084b4cdb7c95", "shape": [ 100 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "322932091bb64b559083df41f6380723142ffa85606836596f3381bffb456144", "shape": [ 400, 100 ] } ], "region": "r:212", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 400 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 100 }, { "annotation": null, "id": "s:232", "inputs": [ "s:233", "s:279" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:211", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 118.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:233", "inputs": [ "s:234" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f603f94b1d517d5138cdeae0c125779455d596b548a58e5cb4d941f97a1a242a", "shape": [ 5, 1 ] } ], "region": "r:213", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 117.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:234", "inputs": [ "s:235", "s:237", "s:257", "s:277", "s:278" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:213", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 45.0, "upper_unbounded": false } }, "width": 5 }, { "annotation": null, "id": "s:235", "inputs": [ "s:236" ], "name": "digit_value", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "23b4c414e1da04e6b50944ee108cf56058564d3fa7601ba5b482a051866afd93", "shape": [ 17, 1 ] } ], "region": null, "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 45.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:236", "inputs": [ "s:100" ], "name": "pp_ones", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "21d56cb7adbc81debe071e26b62a16338d24e5d761ea1155fa5a1f71d4b3f4a0", "shape": [ 34, 17 ] } ], "region": "r:214", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:237", "inputs": [ "s:238" ], "name": "digit_value", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "23b4c414e1da04e6b50944ee108cf56058564d3fa7601ba5b482a051866afd93", "shape": [ 17, 1 ] } ], "region": null, "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 36.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:238", "inputs": [ "s:239" ], "name": "pp_tens", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "b147e672e8eb1f6dd33bc1430113f435acdd9ad22436ba20e281e1360d8054c9", "shape": [ 34, 17 ] } ], "region": "r:215", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:239", "inputs": [ "s:240" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "a6b06fbd738b910030a0847ec7779449c3c076d0975def1ce5d4755431bb9ade", "shape": [ 34 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "bc9b478bb7986ee8a1778fc116c22bb23e7e7352444e62a46c7953e07a2b57bd", "shape": [ 100, 34 ] } ], "region": "r:217", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:240", "inputs": [ "s:102", "s:241" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:218", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:241", "inputs": [ "s:242", "s:245", "s:247" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] } ], "region": "r:219", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 17, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:242", "inputs": [ "s:243", "s:244" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:219", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:243", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:219", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:244", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:219", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:245", "inputs": [ "s:7", "s:246" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:219", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:246", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:219", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:247", "inputs": [ "s:248", "s:201", "s:219" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "ff1247d951578e37ef4f20b3e4a593814438971e903430bf33e180aaeed8c538", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "54e4e0a85bc6a3e7d002e3c35b74ff13c2449639e18520e333933b32d1fb004a", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] } ], "region": "r:221", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 17, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:248", "inputs": [ "s:249", "s:255" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:222", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.0130530234375, "lower_unbounded": false, "upper": 1.0130530234375, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:249", "inputs": [ "s:250" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "56e475e08e4d688e9ca9d3faa1c854f47fc5b6382bdec3ccc4e64114e57ad6e9", "shape": [ 3, 3 ] } ], "region": "r:224", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:250", "inputs": [ "s:251" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "8a31a40ecac0ceb4d87b30bd156ca7a547e8e33dc071454b765fbc777d1c34a1", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "64eec129c8d3f58ee6b7ca145b25e312fa82d3d276db5adaedb59aaebb824885", "shape": [ 3, 3 ] } ], "region": "r:225", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:251", "inputs": [ "s:252" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "10f5b23f39f81bf5063c468e6bb37dbb7cd44d7381dbc28b54c11a333e817e82", "shape": [ 12 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "1ad43993d87c26d5c610bb4df0a3f9b1b03186bb9c69b26284c08db4a57dfa2e", "shape": [ 12, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3ee6949226ed15c57a63c066edaaac5d2a2988e2ba82c56d47f3d1b7f8620845", "shape": [ 12, 3 ] } ], "region": "r:227", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 12 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:252", "inputs": [ "s:253", "s:254" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:226", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.3366749232420485, "lower_unbounded": false, "upper": 2.5033415899087146, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:253", "inputs": [ "s:179" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e4767380eb5e2fc046bce28b8b2a30c81c733be1a56203cd9499066086617f6c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:228", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.3366749232420485, "lower_unbounded": false, "upper": 1.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:254", "inputs": [ "s:253" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:229", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.3366749232420485, "lower_unbounded": false, "upper": 2.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:255", "inputs": [ "s:256" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:230", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.0130530234375, "lower_unbounded": false, "upper": 1.0130530234375, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:256", "inputs": [ "s:249" ], "name": "region_no_lane", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3, 1 ] } ], "region": "r:220", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.0130530234375, "lower_unbounded": false, "upper": 0.013053023437499922, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:257", "inputs": [ "s:258" ], "name": "digit_value", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "23b4c414e1da04e6b50944ee108cf56058564d3fa7601ba5b482a051866afd93", "shape": [ 17, 1 ] } ], "region": null, "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 36.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:258", "inputs": [ "s:259" ], "name": "pp_tens", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "b147e672e8eb1f6dd33bc1430113f435acdd9ad22436ba20e281e1360d8054c9", "shape": [ 34, 17 ] } ], "region": "r:231", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:259", "inputs": [ "s:260" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "a6b06fbd738b910030a0847ec7779449c3c076d0975def1ce5d4755431bb9ade", "shape": [ 34 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "bc9b478bb7986ee8a1778fc116c22bb23e7e7352444e62a46c7953e07a2b57bd", "shape": [ 100, 34 ] } ], "region": "r:233", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:260", "inputs": [ "s:261", "s:166" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:234", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:261", "inputs": [ "s:262", "s:265", "s:267" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] } ], "region": "r:235", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 17, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:262", "inputs": [ "s:263", "s:264" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:235", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:263", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:235", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:264", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:235", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:265", "inputs": [ "s:7", "s:266" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:235", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:266", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:235", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:267", "inputs": [ "s:268", "s:138", "s:163" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "ff1247d951578e37ef4f20b3e4a593814438971e903430bf33e180aaeed8c538", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "54e4e0a85bc6a3e7d002e3c35b74ff13c2449639e18520e333933b32d1fb004a", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] } ], "region": "r:237", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 17, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:268", "inputs": [ "s:269", "s:275" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:238", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.0130530234375, "lower_unbounded": false, "upper": 1.0130530234375, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:269", "inputs": [ "s:270" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "56e475e08e4d688e9ca9d3faa1c854f47fc5b6382bdec3ccc4e64114e57ad6e9", "shape": [ 3, 3 ] } ], "region": "r:240", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:270", "inputs": [ "s:271" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "8a31a40ecac0ceb4d87b30bd156ca7a547e8e33dc071454b765fbc777d1c34a1", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "64eec129c8d3f58ee6b7ca145b25e312fa82d3d276db5adaedb59aaebb824885", "shape": [ 3, 3 ] } ], "region": "r:241", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:271", "inputs": [ "s:272" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "10f5b23f39f81bf5063c468e6bb37dbb7cd44d7381dbc28b54c11a333e817e82", "shape": [ 12 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "1ad43993d87c26d5c610bb4df0a3f9b1b03186bb9c69b26284c08db4a57dfa2e", "shape": [ 12, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3ee6949226ed15c57a63c066edaaac5d2a2988e2ba82c56d47f3d1b7f8620845", "shape": [ 12, 3 ] } ], "region": "r:243", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 12 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:272", "inputs": [ "s:273", "s:274" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:242", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.3366749232420485, "lower_unbounded": false, "upper": 2.5033415899087146, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:273", "inputs": [ "s:115" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e4767380eb5e2fc046bce28b8b2a30c81c733be1a56203cd9499066086617f6c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:244", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.3366749232420485, "lower_unbounded": false, "upper": 1.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:274", "inputs": [ "s:273" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:245", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.3366749232420485, "lower_unbounded": false, "upper": 2.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:275", "inputs": [ "s:276" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:246", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.0130530234375, "lower_unbounded": false, "upper": 1.0130530234375, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:276", "inputs": [ "s:269" ], "name": "region_no_lane", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3, 1 ] } ], "region": "r:236", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.0130530234375, "lower_unbounded": false, "upper": 0.013053023437499922, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:277", "inputs": [ "s:223" ], "name": "digit_value", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "23b4c414e1da04e6b50944ee108cf56058564d3fa7601ba5b482a051866afd93", "shape": [ 17, 1 ] } ], "region": null, "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:278", "inputs": [ "s:223" ], "name": "digit_value", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "23b4c414e1da04e6b50944ee108cf56058564d3fa7601ba5b482a051866afd93", "shape": [ 17, 1 ] } ], "region": null, "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:279", "inputs": [ "s:233" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:247", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 118.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:280", "inputs": [ "s:281" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "81e5bd2ad736b785c26f8ee5ad9ebc0d50450a71dfbb241759a0d378dfca6685", "shape": [ 3, 2 ] } ], "region": "r:248", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:281", "inputs": [ "s:282" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "515d1614aef9d13e7b1592189bacbc4129f9986f5af077d91435419ef5889d28", "shape": [ 9 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "8052157cb96bc0305ae47a1a67c53dc01499b9f7c175846e4687adf3addaf1d9", "shape": [ 9, 6 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "480376c6bf738a0227f2bbf2b3506b7cde209152c0ba9a9077e5527169eb292e", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "30e1bbfda7f7c8c9053a5476e4b927c8d24e47e1e4f277056cf7b9b29f266de8", "shape": [ 9, 3 ] } ], "region": "r:250", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 9 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:282", "inputs": [ "s:283", "s:333" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:251", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "s:283", "inputs": [ "s:284" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "480376c6bf738a0227f2bbf2b3506b7cde209152c0ba9a9077e5527169eb292e", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3f34e6886f39376bb37c08c97f7c480d64bc1e54adb1b9083da6ff148def52c6", "shape": [ 100, 3 ] } ], "region": "r:253", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:284", "inputs": [ "s:285", "s:223" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:254", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:285", "inputs": [ "s:286" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f29de6d677eb31146fcabd1fd447b4654829f0b547c221f0516d422ce631c077", "shape": [ 100, 17 ] } ], "region": "r:255", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:286", "inputs": [ "s:287" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "7a12e561363385e9dfeeab326368731c030ed4b374e7f5897ac819159d2884c5", "shape": [ 100 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c79c7241ec19383f6be1709969fd023c0c6afaedf0eb11cb5b8fd6e8b99856d5", "shape": [ 100, 100 ] } ], "region": "r:257", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 100 }, { "annotation": null, "id": "s:287", "inputs": [ "s:288" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e985c32fd28bde405da64bc903c85250a82a0fa3d2291eca21abbd52b26f503d", "shape": [ 100 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "926cd87c12dbf5fdf0b1c90cf22cbad6b1d3163cec56c0977117efdb2c3efbd3", "shape": [ 100, 100 ] } ], "region": "r:258", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 100 }, { "annotation": null, "id": "s:288", "inputs": [ "s:289" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "5043c9e1d4cb1fea053e49e1d7a8e09157add36a5a1f3d8ee90efd200a572900", "shape": [ 400 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2872925994dc7f63708f8a149b44e2bd8c152caca1b07d0d22d1db16fdfa7b9e", "shape": [ 400, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "79fd15545a108bf1260d68e92162b968655b2a63fc31cc35e91d084b4cdb7c95", "shape": [ 100 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "322932091bb64b559083df41f6380723142ffa85606836596f3381bffb456144", "shape": [ 400, 100 ] } ], "region": "r:260", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 400 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 100 }, { "annotation": null, "id": "s:289", "inputs": [ "s:290", "s:332" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:259", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 46.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:290", "inputs": [ "s:291" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f603f94b1d517d5138cdeae0c125779455d596b548a58e5cb4d941f97a1a242a", "shape": [ 5, 1 ] } ], "region": "r:261", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 45.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:291", "inputs": [ "s:292", "s:328", "s:329", "s:330", "s:331" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:261", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 45.0, "upper_unbounded": false } }, "width": 5 }, { "annotation": null, "id": "s:292", "inputs": [ "s:293" ], "name": "digit_value", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "23b4c414e1da04e6b50944ee108cf56058564d3fa7601ba5b482a051866afd93", "shape": [ 17, 1 ] } ], "region": null, "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 45.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:293", "inputs": [ "s:294" ], "name": "pp_ones", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "21d56cb7adbc81debe071e26b62a16338d24e5d761ea1155fa5a1f71d4b3f4a0", "shape": [ 34, 17 ] } ], "region": "r:262", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:294", "inputs": [ "s:295" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "a6b06fbd738b910030a0847ec7779449c3c076d0975def1ce5d4755431bb9ade", "shape": [ 34 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "bc9b478bb7986ee8a1778fc116c22bb23e7e7352444e62a46c7953e07a2b57bd", "shape": [ 100, 34 ] } ], "region": "r:264", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:295", "inputs": [ "s:296", "s:312" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:265", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:296", "inputs": [ "s:297", "s:300", "s:302" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] } ], "region": "r:266", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 17, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:297", "inputs": [ "s:298", "s:299" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:266", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:298", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:266", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:299", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:266", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:300", "inputs": [ "s:7", "s:301" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:266", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:301", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:266", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:302", "inputs": [ "s:303", "s:138", "s:163" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "ff1247d951578e37ef4f20b3e4a593814438971e903430bf33e180aaeed8c538", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "54e4e0a85bc6a3e7d002e3c35b74ff13c2449639e18520e333933b32d1fb004a", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] } ], "region": "r:268", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 17, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:303", "inputs": [ "s:304", "s:310" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:269", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.0130530234375, "lower_unbounded": false, "upper": 1.0130530234375, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:304", "inputs": [ "s:305" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "56e475e08e4d688e9ca9d3faa1c854f47fc5b6382bdec3ccc4e64114e57ad6e9", "shape": [ 3, 3 ] } ], "region": "r:271", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:305", "inputs": [ "s:306" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "8a31a40ecac0ceb4d87b30bd156ca7a547e8e33dc071454b765fbc777d1c34a1", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "64eec129c8d3f58ee6b7ca145b25e312fa82d3d276db5adaedb59aaebb824885", "shape": [ 3, 3 ] } ], "region": "r:272", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:306", "inputs": [ "s:307" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "10f5b23f39f81bf5063c468e6bb37dbb7cd44d7381dbc28b54c11a333e817e82", "shape": [ 12 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "1ad43993d87c26d5c610bb4df0a3f9b1b03186bb9c69b26284c08db4a57dfa2e", "shape": [ 12, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3ee6949226ed15c57a63c066edaaac5d2a2988e2ba82c56d47f3d1b7f8620845", "shape": [ 12, 3 ] } ], "region": "r:274", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 12 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:307", "inputs": [ "s:308", "s:309" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:273", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.3366749232420485, "lower_unbounded": false, "upper": 3.5033415899087146, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:308", "inputs": [ "s:115" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:275", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.3366749232420485, "lower_unbounded": false, "upper": 2.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:309", "inputs": [ "s:308" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:276", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.3366749232420485, "lower_unbounded": false, "upper": 3.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:310", "inputs": [ "s:311" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:277", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.0130530234375, "lower_unbounded": false, "upper": 1.0130530234375, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:311", "inputs": [ "s:304" ], "name": "region_no_lane", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3, 1 ] } ], "region": "r:267", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.0130530234375, "lower_unbounded": false, "upper": 0.013053023437499922, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:312", "inputs": [ "s:313", "s:316", "s:318" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "7b02e5a4df77d11e9416e3eb5fa9ab42a06b9d00bb0fc3678acb8a680d12f4ba", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "c348445715bb4dcee357ca0840f46d647be91c4535f3a8334095224e709aaa59", "shape": [ 2, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] } ], "region": "r:278", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 17, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:313", "inputs": [ "s:314", "s:315" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:278", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:314", "inputs": [], "name": "prev_value_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:278", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:315", "inputs": [], "name": "recency_lobe_query_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:278", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:316", "inputs": [ "s:7", "s:317" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:278", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:317", "inputs": [], "name": "recency_lobe_key_one", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ], "values": [ 1.0 ] } ], "region": "r:278", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:318", "inputs": [ "s:319", "s:201", "s:219" ], "name": null, "op": "Attn", "parameters": [ { "dtype": "float32", "name": "key_matrix", "sha256": "ff1247d951578e37ef4f20b3e4a593814438971e903430bf33e180aaeed8c538", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "query_matrix", "sha256": "54e4e0a85bc6a3e7d002e3c35b74ff13c2449639e18520e333933b32d1fb004a", "shape": [ 4, 64 ] }, { "dtype": "float32", "name": "value_matrix", "sha256": "3c1942d45e236b90e4df250008e657ca313e15f90a49d8168677211971c866a6", "shape": [ 17, 17 ] } ], "region": "r:280", "scheduling_predecessors": [], "semantics": { "causal": true, "d_qk": 64, "d_v": 17, "rope_base": 500000.0, "rope_d_rot": 64 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:319", "inputs": [ "s:320", "s:326" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:281", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.0130530234375, "lower_unbounded": false, "upper": 1.0130530234375, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:320", "inputs": [ "s:321" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "15ec7bf0b50732b49f8228e07d24365338f9e3ab994b00af08e5a3bffe55fd8b", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "56e475e08e4d688e9ca9d3faa1c854f47fc5b6382bdec3ccc4e64114e57ad6e9", "shape": [ 3, 3 ] } ], "region": "r:283", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:321", "inputs": [ "s:322" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "8a31a40ecac0ceb4d87b30bd156ca7a547e8e33dc071454b765fbc777d1c34a1", "shape": [ 3 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "64eec129c8d3f58ee6b7ca145b25e312fa82d3d276db5adaedb59aaebb824885", "shape": [ 3, 3 ] } ], "region": "r:284", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:322", "inputs": [ "s:323" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "10f5b23f39f81bf5063c468e6bb37dbb7cd44d7381dbc28b54c11a333e817e82", "shape": [ 12 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "1ad43993d87c26d5c610bb4df0a3f9b1b03186bb9c69b26284c08db4a57dfa2e", "shape": [ 12, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3ee6949226ed15c57a63c066edaaac5d2a2988e2ba82c56d47f3d1b7f8620845", "shape": [ 12, 3 ] } ], "region": "r:286", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 12 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:323", "inputs": [ "s:324", "s:325" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:285", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.3366749232420485, "lower_unbounded": false, "upper": 3.5033415899087146, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:324", "inputs": [ "s:179" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:287", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.3366749232420485, "lower_unbounded": false, "upper": 2.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:325", "inputs": [ "s:324" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:288", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.3366749232420485, "lower_unbounded": false, "upper": 3.5033415899087146, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:326", "inputs": [ "s:327" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:289", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.0130530234375, "lower_unbounded": false, "upper": 1.0130530234375, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:327", "inputs": [ "s:320" ], "name": "region_no_lane", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f48af7dd3f3adb7dcc618687a0a2a61b8df98065ca0c180875e8ae65a88e28e4", "shape": [ 3, 1 ] } ], "region": "r:279", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -3.0130530234375, "lower_unbounded": false, "upper": 0.013053023437499922, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:328", "inputs": [ "s:223" ], "name": "digit_value", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "23b4c414e1da04e6b50944ee108cf56058564d3fa7601ba5b482a051866afd93", "shape": [ 17, 1 ] } ], "region": null, "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:329", "inputs": [ "s:223" ], "name": "digit_value", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "23b4c414e1da04e6b50944ee108cf56058564d3fa7601ba5b482a051866afd93", "shape": [ 17, 1 ] } ], "region": null, "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:330", "inputs": [ "s:223" ], "name": "digit_value", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "23b4c414e1da04e6b50944ee108cf56058564d3fa7601ba5b482a051866afd93", "shape": [ 17, 1 ] } ], "region": null, "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:331", "inputs": [ "s:223" ], "name": "digit_value", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "23b4c414e1da04e6b50944ee108cf56058564d3fa7601ba5b482a051866afd93", "shape": [ 17, 1 ] } ], "region": null, "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:332", "inputs": [ "s:290" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:290", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 46.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:333", "inputs": [ "s:334" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "515d1614aef9d13e7b1592189bacbc4129f9986f5af077d91435419ef5889d28", "shape": [ 9 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "8052157cb96bc0305ae47a1a67c53dc01499b9f7c175846e4687adf3addaf1d9", "shape": [ 9, 6 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "480376c6bf738a0227f2bbf2b3506b7cde209152c0ba9a9077e5527169eb292e", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "30e1bbfda7f7c8c9053a5476e4b927c8d24e47e1e4f277056cf7b9b29f266de8", "shape": [ 9, 3 ] } ], "region": "r:292", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 9 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:334", "inputs": [ "s:335", "s:387" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:293", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "s:335", "inputs": [ "s:336" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "515d1614aef9d13e7b1592189bacbc4129f9986f5af077d91435419ef5889d28", "shape": [ 9 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "8052157cb96bc0305ae47a1a67c53dc01499b9f7c175846e4687adf3addaf1d9", "shape": [ 9, 6 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "480376c6bf738a0227f2bbf2b3506b7cde209152c0ba9a9077e5527169eb292e", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "30e1bbfda7f7c8c9053a5476e4b927c8d24e47e1e4f277056cf7b9b29f266de8", "shape": [ 9, 3 ] } ], "region": "r:295", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 9 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:336", "inputs": [ "s:337", "s:360" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:296", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "s:337", "inputs": [ "s:338" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "480376c6bf738a0227f2bbf2b3506b7cde209152c0ba9a9077e5527169eb292e", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3f34e6886f39376bb37c08c97f7c480d64bc1e54adb1b9083da6ff148def52c6", "shape": [ 100, 3 ] } ], "region": "r:298", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:338", "inputs": [ "s:339", "s:359" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:299", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:339", "inputs": [ "s:340" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f29de6d677eb31146fcabd1fd447b4654829f0b547c221f0516d422ce631c077", "shape": [ 100, 17 ] } ], "region": "r:300", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:340", "inputs": [ "s:341" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "7a12e561363385e9dfeeab326368731c030ed4b374e7f5897ac819159d2884c5", "shape": [ 100 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c79c7241ec19383f6be1709969fd023c0c6afaedf0eb11cb5b8fd6e8b99856d5", "shape": [ 100, 100 ] } ], "region": "r:302", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 100 }, { "annotation": null, "id": "s:341", "inputs": [ "s:342" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e985c32fd28bde405da64bc903c85250a82a0fa3d2291eca21abbd52b26f503d", "shape": [ 100 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "926cd87c12dbf5fdf0b1c90cf22cbad6b1d3163cec56c0977117efdb2c3efbd3", "shape": [ 100, 100 ] } ], "region": "r:303", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 100 }, { "annotation": null, "id": "s:342", "inputs": [ "s:343" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "5043c9e1d4cb1fea053e49e1d7a8e09157add36a5a1f3d8ee90efd200a572900", "shape": [ 400 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2872925994dc7f63708f8a149b44e2bd8c152caca1b07d0d22d1db16fdfa7b9e", "shape": [ 400, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "79fd15545a108bf1260d68e92162b968655b2a63fc31cc35e91d084b4cdb7c95", "shape": [ 100 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "322932091bb64b559083df41f6380723142ffa85606836596f3381bffb456144", "shape": [ 400, 100 ] } ], "region": "r:305", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 400 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 100 }, { "annotation": null, "id": "s:343", "inputs": [ "s:344", "s:358" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:304", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 127.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:344", "inputs": [ "s:345" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f603f94b1d517d5138cdeae0c125779455d596b548a58e5cb4d941f97a1a242a", "shape": [ 5, 1 ] } ], "region": "r:306", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 126.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:345", "inputs": [ "s:346", "s:350", "s:354", "s:356", "s:357" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:306", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 45.0, "upper_unbounded": false } }, "width": 5 }, { "annotation": null, "id": "s:346", "inputs": [ "s:347" ], "name": "digit_value", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "23b4c414e1da04e6b50944ee108cf56058564d3fa7601ba5b482a051866afd93", "shape": [ 17, 1 ] } ], "region": null, "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 45.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:347", "inputs": [ "s:348" ], "name": "pp_ones", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "21d56cb7adbc81debe071e26b62a16338d24e5d761ea1155fa5a1f71d4b3f4a0", "shape": [ 34, 17 ] } ], "region": "r:307", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:348", "inputs": [ "s:349" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "a6b06fbd738b910030a0847ec7779449c3c076d0975def1ce5d4755431bb9ade", "shape": [ 34 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "bc9b478bb7986ee8a1778fc116c22bb23e7e7352444e62a46c7953e07a2b57bd", "shape": [ 100, 34 ] } ], "region": "r:309", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:349", "inputs": [ "s:261", "s:312" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:310", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:350", "inputs": [ "s:351" ], "name": "digit_value", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "23b4c414e1da04e6b50944ee108cf56058564d3fa7601ba5b482a051866afd93", "shape": [ 17, 1 ] } ], "region": null, "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 45.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:351", "inputs": [ "s:352" ], "name": "pp_ones", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "21d56cb7adbc81debe071e26b62a16338d24e5d761ea1155fa5a1f71d4b3f4a0", "shape": [ 34, 17 ] } ], "region": "r:311", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:352", "inputs": [ "s:353" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "a6b06fbd738b910030a0847ec7779449c3c076d0975def1ce5d4755431bb9ade", "shape": [ 34 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "bc9b478bb7986ee8a1778fc116c22bb23e7e7352444e62a46c7953e07a2b57bd", "shape": [ 100, 34 ] } ], "region": "r:313", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:353", "inputs": [ "s:296", "s:241" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:314", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:354", "inputs": [ "s:355" ], "name": "digit_value", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "23b4c414e1da04e6b50944ee108cf56058564d3fa7601ba5b482a051866afd93", "shape": [ 17, 1 ] } ], "region": null, "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 36.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:355", "inputs": [ "s:294" ], "name": "pp_tens", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "b147e672e8eb1f6dd33bc1430113f435acdd9ad22436ba20e281e1360d8054c9", "shape": [ 34, 17 ] } ], "region": "r:315", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:356", "inputs": [ "s:223" ], "name": "digit_value", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "23b4c414e1da04e6b50944ee108cf56058564d3fa7601ba5b482a051866afd93", "shape": [ 17, 1 ] } ], "region": null, "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:357", "inputs": [ "s:223" ], "name": "digit_value", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "23b4c414e1da04e6b50944ee108cf56058564d3fa7601ba5b482a051866afd93", "shape": [ 17, 1 ] } ], "region": null, "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 0.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:358", "inputs": [ "s:344" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:316", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 127.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:359", "inputs": [ "s:286" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "6f8067031a2f2304a92ac0120dc3bace57a8632405b8843002c93959dfbaab6f", "shape": [ 100, 17 ] } ], "region": "r:317", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:360", "inputs": [ "s:361" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "480376c6bf738a0227f2bbf2b3506b7cde209152c0ba9a9077e5527169eb292e", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3f34e6886f39376bb37c08c97f7c480d64bc1e54adb1b9083da6ff148def52c6", "shape": [ 100, 3 ] } ], "region": "r:319", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:361", "inputs": [ "s:362", "s:386" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:320", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:362", "inputs": [ "s:363" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f29de6d677eb31146fcabd1fd447b4654829f0b547c221f0516d422ce631c077", "shape": [ 100, 17 ] } ], "region": "r:321", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:363", "inputs": [ "s:364" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "7a12e561363385e9dfeeab326368731c030ed4b374e7f5897ac819159d2884c5", "shape": [ 100 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c79c7241ec19383f6be1709969fd023c0c6afaedf0eb11cb5b8fd6e8b99856d5", "shape": [ 100, 100 ] } ], "region": "r:323", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 100 }, { "annotation": null, "id": "s:364", "inputs": [ "s:365" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e985c32fd28bde405da64bc903c85250a82a0fa3d2291eca21abbd52b26f503d", "shape": [ 100 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "926cd87c12dbf5fdf0b1c90cf22cbad6b1d3163cec56c0977117efdb2c3efbd3", "shape": [ 100, 100 ] } ], "region": "r:324", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 100 }, { "annotation": null, "id": "s:365", "inputs": [ "s:366" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "5043c9e1d4cb1fea053e49e1d7a8e09157add36a5a1f3d8ee90efd200a572900", "shape": [ 400 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2872925994dc7f63708f8a149b44e2bd8c152caca1b07d0d22d1db16fdfa7b9e", "shape": [ 400, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "79fd15545a108bf1260d68e92162b968655b2a63fc31cc35e91d084b4cdb7c95", "shape": [ 100 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "322932091bb64b559083df41f6380723142ffa85606836596f3381bffb456144", "shape": [ 400, 100 ] } ], "region": "r:326", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 400 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 100 }, { "annotation": null, "id": "s:366", "inputs": [ "s:367", "s:385" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:325", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 208.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:367", "inputs": [ "s:368" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f603f94b1d517d5138cdeae0c125779455d596b548a58e5cb4d941f97a1a242a", "shape": [ 5, 1 ] } ], "region": "r:327", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 207.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:368", "inputs": [ "s:369", "s:373", "s:377", "s:379", "s:383" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:327", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 45.0, "upper_unbounded": false } }, "width": 5 }, { "annotation": null, "id": "s:369", "inputs": [ "s:370" ], "name": "digit_value", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "23b4c414e1da04e6b50944ee108cf56058564d3fa7601ba5b482a051866afd93", "shape": [ 17, 1 ] } ], "region": null, "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 45.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:370", "inputs": [ "s:371" ], "name": "pp_ones", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "21d56cb7adbc81debe071e26b62a16338d24e5d761ea1155fa5a1f71d4b3f4a0", "shape": [ 34, 17 ] } ], "region": "r:328", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:371", "inputs": [ "s:372" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "a6b06fbd738b910030a0847ec7779449c3c076d0975def1ce5d4755431bb9ade", "shape": [ 34 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "bc9b478bb7986ee8a1778fc116c22bb23e7e7352444e62a46c7953e07a2b57bd", "shape": [ 100, 34 ] } ], "region": "r:330", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:372", "inputs": [ "s:102", "s:312" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:331", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:373", "inputs": [ "s:374" ], "name": "digit_value", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "23b4c414e1da04e6b50944ee108cf56058564d3fa7601ba5b482a051866afd93", "shape": [ 17, 1 ] } ], "region": null, "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 45.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:374", "inputs": [ "s:375" ], "name": "pp_ones", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "21d56cb7adbc81debe071e26b62a16338d24e5d761ea1155fa5a1f71d4b3f4a0", "shape": [ 34, 17 ] } ], "region": "r:332", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:375", "inputs": [ "s:376" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "a6b06fbd738b910030a0847ec7779449c3c076d0975def1ce5d4755431bb9ade", "shape": [ 34 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "bc9b478bb7986ee8a1778fc116c22bb23e7e7352444e62a46c7953e07a2b57bd", "shape": [ 100, 34 ] } ], "region": "r:334", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:376", "inputs": [ "s:261", "s:241" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:335", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:377", "inputs": [ "s:378" ], "name": "digit_value", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "23b4c414e1da04e6b50944ee108cf56058564d3fa7601ba5b482a051866afd93", "shape": [ 17, 1 ] } ], "region": null, "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 36.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:378", "inputs": [ "s:348" ], "name": "pp_tens", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "b147e672e8eb1f6dd33bc1430113f435acdd9ad22436ba20e281e1360d8054c9", "shape": [ 34, 17 ] } ], "region": "r:336", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:379", "inputs": [ "s:380" ], "name": "digit_value", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "23b4c414e1da04e6b50944ee108cf56058564d3fa7601ba5b482a051866afd93", "shape": [ 17, 1 ] } ], "region": null, "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 45.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:380", "inputs": [ "s:381" ], "name": "pp_ones", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "21d56cb7adbc81debe071e26b62a16338d24e5d761ea1155fa5a1f71d4b3f4a0", "shape": [ 34, 17 ] } ], "region": "r:337", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:381", "inputs": [ "s:382" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "a6b06fbd738b910030a0847ec7779449c3c076d0975def1ce5d4755431bb9ade", "shape": [ 34 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "bc9b478bb7986ee8a1778fc116c22bb23e7e7352444e62a46c7953e07a2b57bd", "shape": [ 100, 34 ] } ], "region": "r:339", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:382", "inputs": [ "s:296", "s:166" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:340", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:383", "inputs": [ "s:384" ], "name": "digit_value", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "23b4c414e1da04e6b50944ee108cf56058564d3fa7601ba5b482a051866afd93", "shape": [ 17, 1 ] } ], "region": null, "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 36.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:384", "inputs": [ "s:352" ], "name": "pp_tens", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "b147e672e8eb1f6dd33bc1430113f435acdd9ad22436ba20e281e1360d8054c9", "shape": [ 34, 17 ] } ], "region": "r:341", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:385", "inputs": [ "s:367" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:342", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 208.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:386", "inputs": [ "s:340" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "6f8067031a2f2304a92ac0120dc3bace57a8632405b8843002c93959dfbaab6f", "shape": [ 100, 17 ] } ], "region": "r:343", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:387", "inputs": [ "s:388" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "515d1614aef9d13e7b1592189bacbc4129f9986f5af077d91435419ef5889d28", "shape": [ 9 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "8052157cb96bc0305ae47a1a67c53dc01499b9f7c175846e4687adf3addaf1d9", "shape": [ 9, 6 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "480376c6bf738a0227f2bbf2b3506b7cde209152c0ba9a9077e5527169eb292e", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "30e1bbfda7f7c8c9053a5476e4b927c8d24e47e1e4f277056cf7b9b29f266de8", "shape": [ 9, 3 ] } ], "region": "r:345", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 9 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:388", "inputs": [ "s:389", "s:410" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:346", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "s:389", "inputs": [ "s:390" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "480376c6bf738a0227f2bbf2b3506b7cde209152c0ba9a9077e5527169eb292e", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3f34e6886f39376bb37c08c97f7c480d64bc1e54adb1b9083da6ff148def52c6", "shape": [ 100, 3 ] } ], "region": "r:348", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:390", "inputs": [ "s:391", "s:409" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:349", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:391", "inputs": [ "s:392" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f29de6d677eb31146fcabd1fd447b4654829f0b547c221f0516d422ce631c077", "shape": [ 100, 17 ] } ], "region": "r:350", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:392", "inputs": [ "s:393" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "7a12e561363385e9dfeeab326368731c030ed4b374e7f5897ac819159d2884c5", "shape": [ 100 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c79c7241ec19383f6be1709969fd023c0c6afaedf0eb11cb5b8fd6e8b99856d5", "shape": [ 100, 100 ] } ], "region": "r:352", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 100 }, { "annotation": null, "id": "s:393", "inputs": [ "s:394" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e985c32fd28bde405da64bc903c85250a82a0fa3d2291eca21abbd52b26f503d", "shape": [ 100 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "926cd87c12dbf5fdf0b1c90cf22cbad6b1d3163cec56c0977117efdb2c3efbd3", "shape": [ 100, 100 ] } ], "region": "r:353", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 100 }, { "annotation": null, "id": "s:394", "inputs": [ "s:395" ], "name": "in_range", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "5043c9e1d4cb1fea053e49e1d7a8e09157add36a5a1f3d8ee90efd200a572900", "shape": [ 400 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2872925994dc7f63708f8a149b44e2bd8c152caca1b07d0d22d1db16fdfa7b9e", "shape": [ 400, 2 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "79fd15545a108bf1260d68e92162b968655b2a63fc31cc35e91d084b4cdb7c95", "shape": [ 100 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "322932091bb64b559083df41f6380723142ffa85606836596f3381bffb456144", "shape": [ 400, 100 ] } ], "region": "r:355", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 400 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.008702015625, hi=1.008702015625))" } ], "claimed_range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 1.008702015625, "upper_unbounded": false } }, "width": 100 }, { "annotation": null, "id": "s:395", "inputs": [ "s:396", "s:408" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:354", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 199.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:396", "inputs": [ "s:397" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f603f94b1d517d5138cdeae0c125779455d596b548a58e5cb4d941f97a1a242a", "shape": [ 5, 1 ] } ], "region": "r:356", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 198.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:397", "inputs": [ "s:398", "s:400", "s:402", "s:404", "s:406" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:356", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 45.0, "upper_unbounded": false } }, "width": 5 }, { "annotation": null, "id": "s:398", "inputs": [ "s:399" ], "name": "digit_value", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "23b4c414e1da04e6b50944ee108cf56058564d3fa7601ba5b482a051866afd93", "shape": [ 17, 1 ] } ], "region": null, "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 45.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:399", "inputs": [ "s:239" ], "name": "pp_ones", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "21d56cb7adbc81debe071e26b62a16338d24e5d761ea1155fa5a1f71d4b3f4a0", "shape": [ 34, 17 ] } ], "region": "r:357", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:400", "inputs": [ "s:401" ], "name": "digit_value", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "23b4c414e1da04e6b50944ee108cf56058564d3fa7601ba5b482a051866afd93", "shape": [ 17, 1 ] } ], "region": null, "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 36.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:401", "inputs": [ "s:371" ], "name": "pp_tens", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "b147e672e8eb1f6dd33bc1430113f435acdd9ad22436ba20e281e1360d8054c9", "shape": [ 34, 17 ] } ], "region": "r:358", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:402", "inputs": [ "s:403" ], "name": "digit_value", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "23b4c414e1da04e6b50944ee108cf56058564d3fa7601ba5b482a051866afd93", "shape": [ 17, 1 ] } ], "region": null, "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 45.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:403", "inputs": [ "s:259" ], "name": "pp_ones", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "21d56cb7adbc81debe071e26b62a16338d24e5d761ea1155fa5a1f71d4b3f4a0", "shape": [ 34, 17 ] } ], "region": "r:359", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:404", "inputs": [ "s:405" ], "name": "digit_value", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "23b4c414e1da04e6b50944ee108cf56058564d3fa7601ba5b482a051866afd93", "shape": [ 17, 1 ] } ], "region": null, "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 36.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:405", "inputs": [ "s:375" ], "name": "pp_tens", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "b147e672e8eb1f6dd33bc1430113f435acdd9ad22436ba20e281e1360d8054c9", "shape": [ 34, 17 ] } ], "region": "r:360", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:406", "inputs": [ "s:407" ], "name": "digit_value", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "23b4c414e1da04e6b50944ee108cf56058564d3fa7601ba5b482a051866afd93", "shape": [ 17, 1 ] } ], "region": null, "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 36.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:407", "inputs": [ "s:381" ], "name": "pp_tens", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "b147e672e8eb1f6dd33bc1430113f435acdd9ad22436ba20e281e1360d8054c9", "shape": [ 34, 17 ] } ], "region": "r:361", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:408", "inputs": [ "s:396" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:362", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 1.0, "lower_unbounded": false, "upper": 199.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:409", "inputs": [ "s:363" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "6f8067031a2f2304a92ac0120dc3bace57a8632405b8843002c93959dfbaab6f", "shape": [ 100, 17 ] } ], "region": "r:363", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:410", "inputs": [ "s:411" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "480376c6bf738a0227f2bbf2b3506b7cde209152c0ba9a9077e5527169eb292e", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3f34e6886f39376bb37c08c97f7c480d64bc1e54adb1b9083da6ff148def52c6", "shape": [ 100, 3 ] } ], "region": "r:365", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:411", "inputs": [ "s:412", "s:413" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:366", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:412", "inputs": [ "s:229" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f29de6d677eb31146fcabd1fd447b4654829f0b547c221f0516d422ce631c077", "shape": [ 100, 17 ] } ], "region": "r:367", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:413", "inputs": [ "s:392" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "6f8067031a2f2304a92ac0120dc3bace57a8632405b8843002c93959dfbaab6f", "shape": [ 100, 17 ] } ], "region": "r:368", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:414", "inputs": [ "s:415" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "region": "r:370", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:415", "inputs": [ "s:416" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:371", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0043510078124998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:416", "inputs": [ "s:417" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "eb9f1ff3e4fa10d12a6ae64cce02175007e98fff40f70e2ba8fdbf8f5a123795", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:373", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:417", "inputs": [ "s:418" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:374", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:418", "inputs": [ "s:39", "s:419" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:374", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:419", "inputs": [ "s:420" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "region": "r:376", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:420", "inputs": [ "s:421" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:377", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:421", "inputs": [ "s:422" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d02095ef9169de77f0d4bcaefe3494a01a6cb5303d8f16874d410303fff7354a", "shape": [ 1, 17 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "region": "r:379", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:422", "inputs": [ "s:423" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9de7dcac2491cd60a9a20fa1b13cbaa6a2ae2547b251a17e2001b747a444d46d", "shape": [ 51, 17 ] } ], "region": "r:381", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:423", "inputs": [ "s:424", "s:426", "s:428" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:381", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 51 }, { "annotation": null, "id": "s:424", "inputs": [ "s:425" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:383", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:425", "inputs": [ "s:46", "s:71" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:382", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:426", "inputs": [ "s:427" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:385", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:427", "inputs": [ "s:74", "s:80" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:384", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:428", "inputs": [ "s:429" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:387", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:429", "inputs": [ "s:83", "s:430" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:386", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:430", "inputs": [ "s:431" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "ed95422d29540bfc7f36e115c60ffa7a8494973d7cfe886d9611f90169997399", "shape": [ 200, 17 ] } ], "region": "r:389", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:431", "inputs": [ "s:412", "s:413", "s:432" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:390", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 36 }, { "annotation": null, "id": "s:432", "inputs": [ "s:433" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "81e5bd2ad736b785c26f8ee5ad9ebc0d50450a71dfbb241759a0d378dfca6685", "shape": [ 3, 2 ] } ], "region": "r:391", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:433", "inputs": [ "s:434" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "515d1614aef9d13e7b1592189bacbc4129f9986f5af077d91435419ef5889d28", "shape": [ 9 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "8052157cb96bc0305ae47a1a67c53dc01499b9f7c175846e4687adf3addaf1d9", "shape": [ 9, 6 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "480376c6bf738a0227f2bbf2b3506b7cde209152c0ba9a9077e5527169eb292e", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "30e1bbfda7f7c8c9053a5476e4b927c8d24e47e1e4f277056cf7b9b29f266de8", "shape": [ 9, 3 ] } ], "region": "r:393", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 9 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:434", "inputs": [ "s:435", "s:437" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:394", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "s:435", "inputs": [ "s:436" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "515d1614aef9d13e7b1592189bacbc4129f9986f5af077d91435419ef5889d28", "shape": [ 9 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "8052157cb96bc0305ae47a1a67c53dc01499b9f7c175846e4687adf3addaf1d9", "shape": [ 9, 6 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "480376c6bf738a0227f2bbf2b3506b7cde209152c0ba9a9077e5527169eb292e", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "30e1bbfda7f7c8c9053a5476e4b927c8d24e47e1e4f277056cf7b9b29f266de8", "shape": [ 9, 3 ] } ], "region": "r:396", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 9 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:436", "inputs": [ "s:283", "s:337" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:397", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "s:437", "inputs": [ "s:438" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "515d1614aef9d13e7b1592189bacbc4129f9986f5af077d91435419ef5889d28", "shape": [ 9 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "8052157cb96bc0305ae47a1a67c53dc01499b9f7c175846e4687adf3addaf1d9", "shape": [ 9, 6 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "480376c6bf738a0227f2bbf2b3506b7cde209152c0ba9a9077e5527169eb292e", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "30e1bbfda7f7c8c9053a5476e4b927c8d24e47e1e4f277056cf7b9b29f266de8", "shape": [ 9, 3 ] } ], "region": "r:399", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 9 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:438", "inputs": [ "s:360", "s:389" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:400", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "s:439", "inputs": [ "s:440" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "region": "r:402", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:440", "inputs": [ "s:441" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:403", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0043510078124998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:441", "inputs": [ "s:442" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1cc95150dbcdc695d9fe82393ccc24788e0bbdb47c092c4100228da7b850f0b9", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:405", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:442", "inputs": [ "s:443" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "8a31a40ecac0ceb4d87b30bd156ca7a547e8e33dc071454b765fbc777d1c34a1", "shape": [ 3, 1 ] } ], "region": "r:406", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.006526511718749961, "lower_unbounded": false, "upper": 3.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:443", "inputs": [ "s:39", "s:419", "s:444" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:406", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:444", "inputs": [ "s:445" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "region": "r:408", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:445", "inputs": [ "s:446" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:409", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:446", "inputs": [ "s:447" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d02095ef9169de77f0d4bcaefe3494a01a6cb5303d8f16874d410303fff7354a", "shape": [ 1, 17 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "region": "r:411", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:447", "inputs": [ "s:448" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9de7dcac2491cd60a9a20fa1b13cbaa6a2ae2547b251a17e2001b747a444d46d", "shape": [ 51, 17 ] } ], "region": "r:413", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:448", "inputs": [ "s:449", "s:465", "s:467" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:413", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 51 }, { "annotation": null, "id": "s:449", "inputs": [ "s:450" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:415", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:450", "inputs": [ "s:46", "s:451" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:414", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:451", "inputs": [ "s:452" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "ed95422d29540bfc7f36e115c60ffa7a8494973d7cfe886d9611f90169997399", "shape": [ 200, 17 ] } ], "region": "r:417", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:452", "inputs": [ "s:453", "s:453", "s:454" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:418", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 36 }, { "annotation": null, "id": "s:453", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ], "values": [ 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "region": "r:419", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:454", "inputs": [ "s:455" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "81e5bd2ad736b785c26f8ee5ad9ebc0d50450a71dfbb241759a0d378dfca6685", "shape": [ 3, 2 ] } ], "region": "r:420", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:455", "inputs": [ "s:456" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "515d1614aef9d13e7b1592189bacbc4129f9986f5af077d91435419ef5889d28", "shape": [ 9 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "8052157cb96bc0305ae47a1a67c53dc01499b9f7c175846e4687adf3addaf1d9", "shape": [ 9, 6 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "480376c6bf738a0227f2bbf2b3506b7cde209152c0ba9a9077e5527169eb292e", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "30e1bbfda7f7c8c9053a5476e4b927c8d24e47e1e4f277056cf7b9b29f266de8", "shape": [ 9, 3 ] } ], "region": "r:422", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 9 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:456", "inputs": [ "s:457", "s:459" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:423", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "s:457", "inputs": [ "s:458" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "480376c6bf738a0227f2bbf2b3506b7cde209152c0ba9a9077e5527169eb292e", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3f34e6886f39376bb37c08c97f7c480d64bc1e54adb1b9083da6ff148def52c6", "shape": [ 100, 3 ] } ], "region": "r:425", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:458", "inputs": [ "s:296", "s:312" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:426", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:459", "inputs": [ "s:460" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "515d1614aef9d13e7b1592189bacbc4129f9986f5af077d91435419ef5889d28", "shape": [ 9 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "8052157cb96bc0305ae47a1a67c53dc01499b9f7c175846e4687adf3addaf1d9", "shape": [ 9, 6 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "480376c6bf738a0227f2bbf2b3506b7cde209152c0ba9a9077e5527169eb292e", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "30e1bbfda7f7c8c9053a5476e4b927c8d24e47e1e4f277056cf7b9b29f266de8", "shape": [ 9, 3 ] } ], "region": "r:428", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 9 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:460", "inputs": [ "s:461", "s:463" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:429", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "s:461", "inputs": [ "s:462" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "480376c6bf738a0227f2bbf2b3506b7cde209152c0ba9a9077e5527169eb292e", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3f34e6886f39376bb37c08c97f7c480d64bc1e54adb1b9083da6ff148def52c6", "shape": [ 100, 3 ] } ], "region": "r:431", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:462", "inputs": [ "s:261", "s:241" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:432", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:463", "inputs": [ "s:464" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "480376c6bf738a0227f2bbf2b3506b7cde209152c0ba9a9077e5527169eb292e", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "3f34e6886f39376bb37c08c97f7c480d64bc1e54adb1b9083da6ff148def52c6", "shape": [ 100, 3 ] } ], "region": "r:434", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:464", "inputs": [ "s:102", "s:166" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:435", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:465", "inputs": [ "s:466" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:437", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:466", "inputs": [ "s:74", "s:80" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:436", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:467", "inputs": [ "s:468" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:439", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:468", "inputs": [ "s:83", "s:469" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:438", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:469", "inputs": [ "s:470" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "ed95422d29540bfc7f36e115c60ffa7a8494973d7cfe886d9611f90169997399", "shape": [ 200, 17 ] } ], "region": "r:441", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:470", "inputs": [ "s:391", "s:409", "s:471" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:442", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 36 }, { "annotation": null, "id": "s:471", "inputs": [ "s:472" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "81e5bd2ad736b785c26f8ee5ad9ebc0d50450a71dfbb241759a0d378dfca6685", "shape": [ 3, 2 ] } ], "region": "r:443", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:472", "inputs": [ "s:473" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "515d1614aef9d13e7b1592189bacbc4129f9986f5af077d91435419ef5889d28", "shape": [ 9 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "8052157cb96bc0305ae47a1a67c53dc01499b9f7c175846e4687adf3addaf1d9", "shape": [ 9, 6 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "480376c6bf738a0227f2bbf2b3506b7cde209152c0ba9a9077e5527169eb292e", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "30e1bbfda7f7c8c9053a5476e4b927c8d24e47e1e4f277056cf7b9b29f266de8", "shape": [ 9, 3 ] } ], "region": "r:445", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 9 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:473", "inputs": [ "s:283", "s:335" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:446", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "s:474", "inputs": [ "s:475" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "region": "r:448", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:475", "inputs": [ "s:476" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:449", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0043510078124998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:476", "inputs": [ "s:477" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0a981cf047d1413d4048a8918c40e5341ed3c1e754371a27df6dbebbc4041fc9", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:451", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:477", "inputs": [ "s:478" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f6bb1294da2f78cd935b01c7656280df5eaa0439e9d97bc03775825a41a508e4", "shape": [ 4, 1 ] } ], "region": "r:452", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.008702015624999948, "lower_unbounded": false, "upper": 4.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:478", "inputs": [ "s:39", "s:419", "s:444", "s:479" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:452", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 4 }, { "annotation": null, "id": "s:479", "inputs": [ "s:480" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "region": "r:454", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:480", "inputs": [ "s:481" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:455", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:481", "inputs": [ "s:482" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d02095ef9169de77f0d4bcaefe3494a01a6cb5303d8f16874d410303fff7354a", "shape": [ 1, 17 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "region": "r:457", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:482", "inputs": [ "s:483" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9de7dcac2491cd60a9a20fa1b13cbaa6a2ae2547b251a17e2001b747a444d46d", "shape": [ 51, 17 ] } ], "region": "r:459", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:483", "inputs": [ "s:484", "s:491", "s:529" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:459", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 51 }, { "annotation": null, "id": "s:484", "inputs": [ "s:485" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:461", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:485", "inputs": [ "s:46", "s:486" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:460", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:486", "inputs": [ "s:487" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "ed95422d29540bfc7f36e115c60ffa7a8494973d7cfe886d9611f90169997399", "shape": [ 200, 17 ] } ], "region": "r:463", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:487", "inputs": [ "s:102", "s:166", "s:488" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:464", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 36 }, { "annotation": null, "id": "s:488", "inputs": [ "s:489" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "81e5bd2ad736b785c26f8ee5ad9ebc0d50450a71dfbb241759a0d378dfca6685", "shape": [ 3, 2 ] } ], "region": "r:465", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:489", "inputs": [ "s:490" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "515d1614aef9d13e7b1592189bacbc4129f9986f5af077d91435419ef5889d28", "shape": [ 9 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "8052157cb96bc0305ae47a1a67c53dc01499b9f7c175846e4687adf3addaf1d9", "shape": [ 9, 6 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "480376c6bf738a0227f2bbf2b3506b7cde209152c0ba9a9077e5527169eb292e", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "30e1bbfda7f7c8c9053a5476e4b927c8d24e47e1e4f277056cf7b9b29f266de8", "shape": [ 9, 3 ] } ], "region": "r:467", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 9 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:490", "inputs": [ "s:457", "s:461" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:468", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "s:491", "inputs": [ "s:492" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:470", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:492", "inputs": [ "s:74", "s:493" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:469", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:493", "inputs": [ "s:494" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "466a4cf2f6f4299d9cc7fd314c658f94d7a2a1dbe112cb598d2ea23fba6d9a08", "shape": [ 200, 17 ] } ], "region": "r:472", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:494", "inputs": [ "s:495", "s:512", "s:514" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:473", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 36 }, { "annotation": null, "id": "s:495", "inputs": [ "s:496" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "3422cbe5c1e8b5b1e016e89ff3387bd4290361fdb17782c50e53c187cf575c5d", "shape": [ 34, 35 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e0ebe36501a3a3bd49b9c2eca7f757e673188e21386e2ef140b112922c14d71b", "shape": [ 34, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7bd2d0249f58c2a4d46659f07872f284b01435b9afd6b0e16d9551cfac29acb6", "shape": [ 34, 35 ] } ], "region": "r:475", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 34 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:496", "inputs": [ "s:497", "s:102", "s:166" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:474", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 35 }, { "annotation": null, "id": "s:497", "inputs": [ "s:498" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "25870a3818cab31267591ff755d69e74c8435d6e091e4ff7c8ca9945f2f3285c", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:477", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:498", "inputs": [ "s:499" ], "name": "cmp_score", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9e115a07ab905736a35a795d4e6bc3563c8cb99ceb53c43840d9195037bc779c", "shape": [ 6, 1 ] } ], "region": null, "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -13.0565631015625, "lower_unbounded": false, "upper": 13.0565631015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:499", "inputs": [ "s:500", "s:503", "s:504", "s:507", "s:508", "s:511" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:478", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "s:500", "inputs": [ "s:501" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "28c4f7df3425ac5087c2632d76786a4afaaf06906eeafb89d8d36ba650e3f24b", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "aced851223ade84128237ff20c2acca7ac1102b3d2dfea9f7fcfb3eef3457a01", "shape": [ 2, 1 ] } ], "region": "r:480", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.00217550390625, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:501", "inputs": [ "s:502" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "64e48d62679d2010a89384ca9bac6ee4bcddfdfb5c838dc5aef8b4e8a40fbec8", "shape": [ 100, 1 ] } ], "region": "r:482", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0, hi=1.0))" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:502", "inputs": [ "s:102", "s:166" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:483", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:503", "inputs": [ "s:501" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "25870a3818cab31267591ff755d69e74c8435d6e091e4ff7c8ca9945f2f3285c", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "a1f63f4bc9d1e1abb868bb7c396f4b79db414fdabb2519e660a356486e507089", "shape": [ 2, 1 ] } ], "region": "r:485", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.00217550390625, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:504", "inputs": [ "s:505" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "28c4f7df3425ac5087c2632d76786a4afaaf06906eeafb89d8d36ba650e3f24b", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "aced851223ade84128237ff20c2acca7ac1102b3d2dfea9f7fcfb3eef3457a01", "shape": [ 2, 1 ] } ], "region": "r:487", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.00217550390625, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:505", "inputs": [ "s:506" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "64e48d62679d2010a89384ca9bac6ee4bcddfdfb5c838dc5aef8b4e8a40fbec8", "shape": [ 100, 1 ] } ], "region": "r:489", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0, hi=1.0))" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:506", "inputs": [ "s:261", "s:241" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:490", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:507", "inputs": [ "s:505" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "25870a3818cab31267591ff755d69e74c8435d6e091e4ff7c8ca9945f2f3285c", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "a1f63f4bc9d1e1abb868bb7c396f4b79db414fdabb2519e660a356486e507089", "shape": [ 2, 1 ] } ], "region": "r:492", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.00217550390625, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:508", "inputs": [ "s:509" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "28c4f7df3425ac5087c2632d76786a4afaaf06906eeafb89d8d36ba650e3f24b", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "aced851223ade84128237ff20c2acca7ac1102b3d2dfea9f7fcfb3eef3457a01", "shape": [ 2, 1 ] } ], "region": "r:494", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.00217550390625, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:509", "inputs": [ "s:510" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "64e48d62679d2010a89384ca9bac6ee4bcddfdfb5c838dc5aef8b4e8a40fbec8", "shape": [ 100, 1 ] } ], "region": "r:496", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0, hi=1.0))" } ], "claimed_range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:510", "inputs": [ "s:296", "s:312" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:497", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:511", "inputs": [ "s:509" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "25870a3818cab31267591ff755d69e74c8435d6e091e4ff7c8ca9945f2f3285c", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "a1f63f4bc9d1e1abb868bb7c396f4b79db414fdabb2519e660a356486e507089", "shape": [ 2, 1 ] } ], "region": "r:499", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-0.00217550390625, hi=1.00217550390625))" } ], "claimed_range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false }, "integer": false, "range": { "lower": -0.00217550390625, "lower_unbounded": false, "upper": 1.00217550390625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:512", "inputs": [ "s:513" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "3422cbe5c1e8b5b1e016e89ff3387bd4290361fdb17782c50e53c187cf575c5d", "shape": [ 34, 35 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e0ebe36501a3a3bd49b9c2eca7f757e673188e21386e2ef140b112922c14d71b", "shape": [ 34, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7bd2d0249f58c2a4d46659f07872f284b01435b9afd6b0e16d9551cfac29acb6", "shape": [ 34, 35 ] } ], "region": "r:501", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 34 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:513", "inputs": [ "s:497", "s:166", "s:102" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:500", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 35 }, { "annotation": null, "id": "s:514", "inputs": [ "s:515" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "81e5bd2ad736b785c26f8ee5ad9ebc0d50450a71dfbb241759a0d378dfca6685", "shape": [ 3, 2 ] } ], "region": "r:502", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:515", "inputs": [ "s:516" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "515d1614aef9d13e7b1592189bacbc4129f9986f5af077d91435419ef5889d28", "shape": [ 9 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "8052157cb96bc0305ae47a1a67c53dc01499b9f7c175846e4687adf3addaf1d9", "shape": [ 9, 6 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "480376c6bf738a0227f2bbf2b3506b7cde209152c0ba9a9077e5527169eb292e", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "30e1bbfda7f7c8c9053a5476e4b927c8d24e47e1e4f277056cf7b9b29f266de8", "shape": [ 9, 3 ] } ], "region": "r:504", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 9 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:516", "inputs": [ "s:517", "s:523" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:505", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 6 }, { "annotation": null, "id": "s:517", "inputs": [ "s:518" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "480376c6bf738a0227f2bbf2b3506b7cde209152c0ba9a9077e5527169eb292e", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "51206ad98c150026d0995579546dd690743e9afd4918c78cb6e674f13df9da5b", "shape": [ 100, 3 ] } ], "region": "r:507", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:518", "inputs": [ "s:519", "s:521" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:508", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:519", "inputs": [ "s:520" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "3422cbe5c1e8b5b1e016e89ff3387bd4290361fdb17782c50e53c187cf575c5d", "shape": [ 34, 35 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e0ebe36501a3a3bd49b9c2eca7f757e673188e21386e2ef140b112922c14d71b", "shape": [ 34, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7bd2d0249f58c2a4d46659f07872f284b01435b9afd6b0e16d9551cfac29acb6", "shape": [ 34, 35 ] } ], "region": "r:510", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 34 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:520", "inputs": [ "s:497", "s:296", "s:312" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:509", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 35 }, { "annotation": null, "id": "s:521", "inputs": [ "s:522" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "3422cbe5c1e8b5b1e016e89ff3387bd4290361fdb17782c50e53c187cf575c5d", "shape": [ 34, 35 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e0ebe36501a3a3bd49b9c2eca7f757e673188e21386e2ef140b112922c14d71b", "shape": [ 34, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7bd2d0249f58c2a4d46659f07872f284b01435b9afd6b0e16d9551cfac29acb6", "shape": [ 34, 35 ] } ], "region": "r:512", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 34 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:522", "inputs": [ "s:497", "s:312", "s:296" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:511", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 35 }, { "annotation": null, "id": "s:523", "inputs": [ "s:524" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6907395ae3c2926830c7a2b5c73935086066eb245b8c4606bcf8c89e8fb3a332", "shape": [ 100 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "a86d6c4d682eb4c6ef3ef2cd786f7649ac67ca93a8f8f889259f268479a130e3", "shape": [ 100, 34 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "480376c6bf738a0227f2bbf2b3506b7cde209152c0ba9a9077e5527169eb292e", "shape": [ 3 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "51206ad98c150026d0995579546dd690743e9afd4918c78cb6e674f13df9da5b", "shape": [ 100, 3 ] } ], "region": "r:514", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 100 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 3 }, { "annotation": null, "id": "s:524", "inputs": [ "s:525", "s:527" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:515", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 34 }, { "annotation": null, "id": "s:525", "inputs": [ "s:526" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "3422cbe5c1e8b5b1e016e89ff3387bd4290361fdb17782c50e53c187cf575c5d", "shape": [ 34, 35 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e0ebe36501a3a3bd49b9c2eca7f757e673188e21386e2ef140b112922c14d71b", "shape": [ 34, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7bd2d0249f58c2a4d46659f07872f284b01435b9afd6b0e16d9551cfac29acb6", "shape": [ 34, 35 ] } ], "region": "r:517", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 34 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:526", "inputs": [ "s:497", "s:261", "s:241" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:516", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 35 }, { "annotation": null, "id": "s:527", "inputs": [ "s:528" ], "name": "select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "3422cbe5c1e8b5b1e016e89ff3387bd4290361fdb17782c50e53c187cf575c5d", "shape": [ 34, 35 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "e0ebe36501a3a3bd49b9c2eca7f757e673188e21386e2ef140b112922c14d71b", "shape": [ 34, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "b707241545a346265aab1ffb32ff64b55bf8f8dc1b56a46ef33ce3d15db11d33", "shape": [ 34 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7bd2d0249f58c2a4d46659f07872f284b01435b9afd6b0e16d9551cfac29acb6", "shape": [ 34, 35 ] } ], "region": "r:519", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 34 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:528", "inputs": [ "s:497", "s:241", "s:261" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:518", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 35 }, { "annotation": null, "id": "s:529", "inputs": [ "s:530" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:521", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:530", "inputs": [ "s:83", "s:531" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:520", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:531", "inputs": [ "s:532" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "ed95422d29540bfc7f36e115c60ffa7a8494973d7cfe886d9611f90169997399", "shape": [ 200, 17 ] } ], "region": "r:523", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:532", "inputs": [ "s:362", "s:386", "s:533" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:524", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 36 }, { "annotation": null, "id": "s:533", "inputs": [ "s:435" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "81e5bd2ad736b785c26f8ee5ad9ebc0d50450a71dfbb241759a0d378dfca6685", "shape": [ 3, 2 ] } ], "region": "r:525", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:534", "inputs": [ "s:535" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "region": "r:527", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:535", "inputs": [ "s:536" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:528", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0043510078124998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:536", "inputs": [ "s:537" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "db8d1a556ec9527f63930661298f4b8607cfcecb3c364235751acfef1ec8ed1a", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:530", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:537", "inputs": [ "s:538" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "f603f94b1d517d5138cdeae0c125779455d596b548a58e5cb4d941f97a1a242a", "shape": [ 5, 1 ] } ], "region": "r:531", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.010877519531249935, "lower_unbounded": false, "upper": 5.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:538", "inputs": [ "s:39", "s:419", "s:444", "s:479", "s:539" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:531", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 5 }, { "annotation": null, "id": "s:539", "inputs": [ "s:540" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "region": "r:533", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:540", "inputs": [ "s:541" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:534", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:541", "inputs": [ "s:542" ], "name": "equals_vector", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d02095ef9169de77f0d4bcaefe3494a01a6cb5303d8f16874d410303fff7354a", "shape": [ 1, 17 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "4efb856a85ce3cd60020d80d64475630ab731c6b6a31198ed4417c6dfd1829c2", "shape": [ 1, 1 ] } ], "region": "r:536", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 1 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:542", "inputs": [ "s:543" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9de7dcac2491cd60a9a20fa1b13cbaa6a2ae2547b251a17e2001b747a444d46d", "shape": [ 51, 17 ] } ], "region": "r:538", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:543", "inputs": [ "s:544", "s:549", "s:554" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:538", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 51 }, { "annotation": null, "id": "s:544", "inputs": [ "s:545" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:540", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:545", "inputs": [ "s:46", "s:546" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:539", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:546", "inputs": [ "s:547" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "ed95422d29540bfc7f36e115c60ffa7a8494973d7cfe886d9611f90169997399", "shape": [ 200, 17 ] } ], "region": "r:542", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:547", "inputs": [ "s:261", "s:241", "s:548" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:543", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 36 }, { "annotation": null, "id": "s:548", "inputs": [ "s:457" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "81e5bd2ad736b785c26f8ee5ad9ebc0d50450a71dfbb241759a0d378dfca6685", "shape": [ 3, 2 ] } ], "region": "r:544", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:549", "inputs": [ "s:550" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:546", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:550", "inputs": [ "s:74", "s:551" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:545", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:551", "inputs": [ "s:552" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "466a4cf2f6f4299d9cc7fd314c658f94d7a2a1dbe112cb598d2ea23fba6d9a08", "shape": [ 200, 17 ] } ], "region": "r:548", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:552", "inputs": [ "s:525", "s:527", "s:553" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:549", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 36 }, { "annotation": null, "id": "s:553", "inputs": [ "s:517" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "81e5bd2ad736b785c26f8ee5ad9ebc0d50450a71dfbb241759a0d378dfca6685", "shape": [ 3, 2 ] } ], "region": "r:550", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:554", "inputs": [ "s:555" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:552", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:555", "inputs": [ "s:83", "s:556" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:551", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:556", "inputs": [ "s:557" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "ed95422d29540bfc7f36e115c60ffa7a8494973d7cfe886d9611f90169997399", "shape": [ 200, 17 ] } ], "region": "r:554", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:557", "inputs": [ "s:339", "s:359", "s:558" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:555", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 36 }, { "annotation": null, "id": "s:558", "inputs": [ "s:283" ], "name": "onehot_lookup_select", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "af5570f5a1810b7af78caf4bc70a660f0df51e42baf91d4de5b2328de0e83dfc", "shape": [ 2 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "81e5bd2ad736b785c26f8ee5ad9ebc0d50450a71dfbb241759a0d378dfca6685", "shape": [ 3, 2 ] } ], "region": "r:556", "scheduling_predecessors": [], "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:559", "inputs": [ "s:560" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "fedcca07b1ccdacce623cb6d8afdeed0314e8508d763e228871f18d4e0ebb7c4", "shape": [ 1, 1 ] } ], "region": "r:557", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.013053023437499922, "lower_unbounded": false, "upper": 6.013053023437499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:560", "inputs": [ "s:561" ], "name": "multiply_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d99e58435243d9fef9c88273b8d553b4fba4d0baf8009d29eae74fa99e0d9f57", "shape": [ 1, 1 ] } ], "region": "r:559", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.002175503906249987, "lower_unbounded": false, "upper": 1.0021755039062499, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:561", "inputs": [ "s:562" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:560", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -0.004351007812499974, "lower_unbounded": false, "upper": 2.0043510078124998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:562", "inputs": [ "s:563" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:563", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:563", "inputs": [ "s:564" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:564", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.0043510078124998, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:564", "inputs": [ "s:74", "s:565" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:564", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:565", "inputs": [ "s:497" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "6d5daefce1c28efa3aa95fc79e933a5c48d297c21e1b4a16b440e5cd7d4b53f3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "f41491c367f49e7206c45724651464310fcb7d17e3059779d1c95aa901f3fce1", "shape": [ 2, 1 ] } ], "region": "r:567", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:566", "inputs": [ "s:31" ], "name": "add_const", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "e00e5eb9444182f352323374ef4e08ebcb784725fdd4fd612d7730540b3e0c8c", "shape": [ 1, 1 ] } ], "region": "r:568", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.9760694570312501, "lower_unbounded": false, "upper": 12.023930542968749, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:567", "inputs": [ "s:42", "s:422", "s:447", "s:482", "s:542", "s:568", "s:585", "s:585", "s:585", "s:585", "s:585", "s:585" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:569", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "s:568", "inputs": [ "s:569" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9de7dcac2491cd60a9a20fa1b13cbaa6a2ae2547b251a17e2001b747a444d46d", "shape": [ 51, 17 ] } ], "region": "r:571", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:569", "inputs": [ "s:570", "s:575", "s:580" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:571", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 51 }, { "annotation": null, "id": "s:570", "inputs": [ "s:571" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:573", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:571", "inputs": [ "s:46", "s:572" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:572", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:572", "inputs": [ "s:573" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "ed95422d29540bfc7f36e115c60ffa7a8494973d7cfe886d9611f90169997399", "shape": [ 200, 17 ] } ], "region": "r:575", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:573", "inputs": [ "s:296", "s:312", "s:574" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:576", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 36 }, { "annotation": null, "id": "s:574", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "434b26042aff3fb844a4c4c6be0d81a079b0ce84cfb8190679024404e5dc4822", "shape": [ 2 ], "values": [ 1.0, 0.0 ] } ], "region": "r:577", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:575", "inputs": [ "s:576" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:579", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:576", "inputs": [ "s:74", "s:577" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:578", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:577", "inputs": [ "s:578" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "466a4cf2f6f4299d9cc7fd314c658f94d7a2a1dbe112cb598d2ea23fba6d9a08", "shape": [ 200, 17 ] } ], "region": "r:581", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:578", "inputs": [ "s:519", "s:521", "s:579" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:582", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 36 }, { "annotation": null, "id": "s:579", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "434b26042aff3fb844a4c4c6be0d81a079b0ce84cfb8190679024404e5dc4822", "shape": [ 2 ], "values": [ 1.0, 0.0 ] } ], "region": "r:583", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:580", "inputs": [ "s:581" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:585", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:581", "inputs": [ "s:83", "s:582" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:584", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:582", "inputs": [ "s:583" ], "name": "onehot_lookup", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "c4bcea91be8922aa61721c13967b4825055db2814ade8458759b0ff9735abca1", "shape": [ 200 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "d26f4b83b033d74a84a0874ffafee369c16677e8399ba116ffc6dac3662cb854", "shape": [ 200, 36 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "af29918470ea37a3d7c548efa0c2c36661668bed1a4230920790f4ebd35b7634", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "ed95422d29540bfc7f36e115c60ffa7a8494973d7cfe886d9611f90169997399", "shape": [ 200, 17 ] } ], "region": "r:587", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 200 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:583", "inputs": [ "s:285", "s:223", "s:584" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:588", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 36 }, { "annotation": null, "id": "s:584", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "434b26042aff3fb844a4c4c6be0d81a079b0ce84cfb8190679024404e5dc4822", "shape": [ 2 ], "values": [ 1.0, 0.0 ] } ], "region": "r:589", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:585", "inputs": [], "name": "remove_leading_0s_sign", "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "4ef898bcd05c1b46ff5a572b9c0da67abd133f76ada08aa6e708d5322587027c", "shape": [ 17 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] } ], "region": "r:590", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:586", "inputs": [ "s:587" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:592", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=33.45360234820311))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 33.45360234820311, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 33.45360234820311, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:587", "inputs": [ "s:588", "s:594" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:591", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 33.45360234820311, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:588", "inputs": [ "s:589" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:595", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:589", "inputs": [ "s:590" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:596", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:590", "inputs": [ "s:591", "s:592" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:596", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:591", "inputs": [ "s:18" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "28c4f7df3425ac5087c2632d76786a4afaaf06906eeafb89d8d36ba650e3f24b", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:598", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:592", "inputs": [ "s:593" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "9bc792642836b330e7b77ccfcb99d0ea4176f98a05cc12439b285610ead907ae", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:600", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:593", "inputs": [ "s:18" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:601", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -9.50850981139363, "lower_unbounded": false, "upper": 0.3418431447269631, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:594", "inputs": [ "s:595" ], "name": "remove_leading_0s_sum", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d7c028b7eda356619d577d7e3dfef5da2267e2c58f989f7adcb5316c4793dd4c", "shape": [ 204, 17 ] } ], "region": "r:28", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 33.45360234820311, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:595", "inputs": [ "s:596" ], "name": "broadcast_select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2c93ccfa935dc7b48ae36ca9b98d2c3e3e50962957d9f3b464865f5c129d7072", "shape": [ 204, 216 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "08720f852c879614691b2c33f2980888e7cdf29871f0a4e9e60e7446758bfc2e", "shape": [ 204, 204 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7e26b5245da34ef2eb9e8e57159215ac9a011c4de6a4aa42cdbfb51763b7de8f", "shape": [ 204, 216 ] } ], "region": "r:603", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 204 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0149999999999997))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "s:596", "inputs": [ "s:29", "s:597" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:602", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 216 }, { "annotation": null, "id": "s:597", "inputs": [ "s:422", "s:447", "s:482", "s:542", "s:568", "s:598", "s:42", "s:422", "s:447", "s:482", "s:542", "s:568" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:604", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "s:598", "inputs": [ "s:599" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9de7dcac2491cd60a9a20fa1b13cbaa6a2ae2547b251a17e2001b747a444d46d", "shape": [ 51, 17 ] } ], "region": "r:606", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:599", "inputs": [ "s:600", "s:603", "s:606" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:606", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 51 }, { "annotation": null, "id": "s:600", "inputs": [ "s:601" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:608", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:601", "inputs": [ "s:46", "s:602" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:607", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:602", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "0bbff7d224d8ac368f7ae20a16cb4ecb95004d3ca9ad72eb7433632c0fab6028", "shape": [ 17 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0 ] } ], "region": "r:609", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:603", "inputs": [ "s:604" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:611", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:604", "inputs": [ "s:74", "s:605" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:610", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:605", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "0bbff7d224d8ac368f7ae20a16cb4ecb95004d3ca9ad72eb7433632c0fab6028", "shape": [ 17 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0 ] } ], "region": "r:612", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:606", "inputs": [ "s:607" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:614", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:607", "inputs": [ "s:83", "s:608" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:613", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:608", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "0bbff7d224d8ac368f7ae20a16cb4ecb95004d3ca9ad72eb7433632c0fab6028", "shape": [ 17 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0 ] } ], "region": "r:615", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:609", "inputs": [ "s:610" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:617", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=27.371129193984366))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 27.371129193984366, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 27.371129193984366, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:610", "inputs": [ "s:611", "s:617" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:616", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 27.371129193984366, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:611", "inputs": [ "s:612" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:620", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:612", "inputs": [ "s:613" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:621", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:613", "inputs": [ "s:614", "s:615" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:621", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:614", "inputs": [ "s:18" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "eb9f1ff3e4fa10d12a6ae64cce02175007e98fff40f70e2ba8fdbf8f5a123795", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:623", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:615", "inputs": [ "s:616" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "ed53dbef9c852a8229e4b386e20627c3a89599c7583ec50379471fb4415bc5b3", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:625", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:616", "inputs": [ "s:18" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:626", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -9.50850981139363, "lower_unbounded": false, "upper": 0.3418431447269631, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:617", "inputs": [ "s:618" ], "name": "remove_leading_0s_sum", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d7c028b7eda356619d577d7e3dfef5da2267e2c58f989f7adcb5316c4793dd4c", "shape": [ 204, 17 ] } ], "region": "r:28", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 27.371129193984366, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:618", "inputs": [ "s:619" ], "name": "broadcast_select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2c93ccfa935dc7b48ae36ca9b98d2c3e3e50962957d9f3b464865f5c129d7072", "shape": [ 204, 216 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "08720f852c879614691b2c33f2980888e7cdf29871f0a4e9e60e7446758bfc2e", "shape": [ 204, 204 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7e26b5245da34ef2eb9e8e57159215ac9a011c4de6a4aa42cdbfb51763b7de8f", "shape": [ 204, 216 ] } ], "region": "r:628", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 204 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0149999999999997))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "s:619", "inputs": [ "s:29", "s:620" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:627", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 216 }, { "annotation": null, "id": "s:620", "inputs": [ "s:447", "s:482", "s:542", "s:568", "s:598", "s:621", "s:422", "s:447", "s:482", "s:542", "s:568", "s:598" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:629", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "s:621", "inputs": [ "s:622" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "9de7dcac2491cd60a9a20fa1b13cbaa6a2ae2547b251a17e2001b747a444d46d", "shape": [ 51, 17 ] } ], "region": "r:631", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:622", "inputs": [ "s:623", "s:625", "s:627" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:631", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.005, "upper_unbounded": false } }, "width": 51 }, { "annotation": null, "id": "s:623", "inputs": [ "s:624" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:633", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:624", "inputs": [ "s:46", "s:602" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:632", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:625", "inputs": [ "s:626" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:635", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:626", "inputs": [ "s:74", "s:605" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:634", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:627", "inputs": [ "s:628" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:637", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=1.0))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:628", "inputs": [ "s:83", "s:608" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:636", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:629", "inputs": [ "s:630" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:639", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=21.28865603976562))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 21.28865603976562, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 21.28865603976562, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:630", "inputs": [ "s:631", "s:637" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:638", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 21.28865603976562, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:631", "inputs": [ "s:632" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:642", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:632", "inputs": [ "s:633" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:643", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:633", "inputs": [ "s:634", "s:635" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:643", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:634", "inputs": [ "s:18" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1cc95150dbcdc695d9fe82393ccc24788e0bbdb47c092c4100228da7b850f0b9", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:645", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:635", "inputs": [ "s:636" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "9dfa5cc0a4093ad092e8327b9861cf4f1595c50fa4b3f12efc578b78fe5afdd2", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:647", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:636", "inputs": [ "s:18" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:648", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -9.50850981139363, "lower_unbounded": false, "upper": 0.3418431447269631, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:637", "inputs": [ "s:638" ], "name": "remove_leading_0s_sum", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d7c028b7eda356619d577d7e3dfef5da2267e2c58f989f7adcb5316c4793dd4c", "shape": [ 204, 17 ] } ], "region": "r:28", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 21.28865603976562, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:638", "inputs": [ "s:639" ], "name": "broadcast_select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2c93ccfa935dc7b48ae36ca9b98d2c3e3e50962957d9f3b464865f5c129d7072", "shape": [ 204, 216 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "08720f852c879614691b2c33f2980888e7cdf29871f0a4e9e60e7446758bfc2e", "shape": [ 204, 204 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7e26b5245da34ef2eb9e8e57159215ac9a011c4de6a4aa42cdbfb51763b7de8f", "shape": [ 204, 216 ] } ], "region": "r:650", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 204 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0149999999999997))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "s:639", "inputs": [ "s:29", "s:640" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:649", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 216 }, { "annotation": null, "id": "s:640", "inputs": [ "s:482", "s:542", "s:568", "s:598", "s:621", "s:621", "s:447", "s:482", "s:542", "s:568", "s:598", "s:621" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:651", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "s:641", "inputs": [ "s:642" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:653", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=21.288656039765623))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 21.288656039765623, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 21.288656039765623, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:642", "inputs": [ "s:643", "s:649" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:652", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 21.288656039765623, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:643", "inputs": [ "s:644" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:656", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:644", "inputs": [ "s:645" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:657", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:645", "inputs": [ "s:646", "s:647" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:657", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:646", "inputs": [ "s:18" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0a981cf047d1413d4048a8918c40e5341ed3c1e754371a27df6dbebbc4041fc9", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:659", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:647", "inputs": [ "s:648" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "25eab693bdf7c6ad20a03812dd32b8a7ba27dab587478a5dc1c8b657174ae97d", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:661", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:648", "inputs": [ "s:18" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:662", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -9.50850981139363, "lower_unbounded": false, "upper": 0.3418431447269631, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:649", "inputs": [ "s:650" ], "name": "remove_leading_0s_sum", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d7c028b7eda356619d577d7e3dfef5da2267e2c58f989f7adcb5316c4793dd4c", "shape": [ 204, 17 ] } ], "region": "r:28", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 21.288656039765623, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:650", "inputs": [ "s:651" ], "name": "broadcast_select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2c93ccfa935dc7b48ae36ca9b98d2c3e3e50962957d9f3b464865f5c129d7072", "shape": [ 204, 216 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "08720f852c879614691b2c33f2980888e7cdf29871f0a4e9e60e7446758bfc2e", "shape": [ 204, 204 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7e26b5245da34ef2eb9e8e57159215ac9a011c4de6a4aa42cdbfb51763b7de8f", "shape": [ 204, 216 ] } ], "region": "r:664", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 204 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0149999999999997))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "s:651", "inputs": [ "s:29", "s:652" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:663", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 216 }, { "annotation": null, "id": "s:652", "inputs": [ "s:542", "s:568", "s:598", "s:621", "s:621", "s:621", "s:482", "s:542", "s:568", "s:598", "s:621", "s:621" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:665", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "s:653", "inputs": [ "s:654" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:667", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=27.371129193984373))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 27.371129193984373, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 27.371129193984373, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:654", "inputs": [ "s:655", "s:661" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:666", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 27.371129193984373, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:655", "inputs": [ "s:656" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:670", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:656", "inputs": [ "s:657" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:671", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:657", "inputs": [ "s:658", "s:659" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:671", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:658", "inputs": [ "s:18" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "db8d1a556ec9527f63930661298f4b8607cfcecb3c364235751acfef1ec8ed1a", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:673", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:659", "inputs": [ "s:660" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "64f403ba374325ca540cb27c0375a0ea1dc20ecaeb654fcbe8bea2b756826dab", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:675", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:660", "inputs": [ "s:18" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:676", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -9.50850981139363, "lower_unbounded": false, "upper": 0.3418431447269631, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:661", "inputs": [ "s:662" ], "name": "remove_leading_0s_sum", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d7c028b7eda356619d577d7e3dfef5da2267e2c58f989f7adcb5316c4793dd4c", "shape": [ 204, 17 ] } ], "region": "r:28", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 27.371129193984373, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:662", "inputs": [ "s:663" ], "name": "broadcast_select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2c93ccfa935dc7b48ae36ca9b98d2c3e3e50962957d9f3b464865f5c129d7072", "shape": [ 204, 216 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "08720f852c879614691b2c33f2980888e7cdf29871f0a4e9e60e7446758bfc2e", "shape": [ 204, 204 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7e26b5245da34ef2eb9e8e57159215ac9a011c4de6a4aa42cdbfb51763b7de8f", "shape": [ 204, 216 ] } ], "region": "r:678", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 204 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0149999999999997))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "s:663", "inputs": [ "s:29", "s:664" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:677", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 216 }, { "annotation": null, "id": "s:664", "inputs": [ "s:568", "s:598", "s:621", "s:621", "s:621", "s:621", "s:542", "s:568", "s:598", "s:621", "s:621", "s:621" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:679", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "s:665", "inputs": [ "s:666" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:681", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=33.45360234820312))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 33.45360234820312, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 33.45360234820312, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:666", "inputs": [ "s:667", "s:673" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:680", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 33.45360234820312, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:667", "inputs": [ "s:668" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:684", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:668", "inputs": [ "s:669" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:685", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:669", "inputs": [ "s:670", "s:671" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:685", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:670", "inputs": [ "s:18" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "b8bb7a27e55e02d67b43582292d3134e5715248dc3f4a54dac7e1c71519551b5", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:687", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:671", "inputs": [ "s:672" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "df1b487fa89261e6181b4f1197a2248f295f75619fa52d6c77c2a438a9f5862d", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:689", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:672", "inputs": [ "s:18" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:690", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -9.50850981139363, "lower_unbounded": false, "upper": 0.3418431447269631, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:673", "inputs": [ "s:674" ], "name": "remove_leading_0s_sum", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d7c028b7eda356619d577d7e3dfef5da2267e2c58f989f7adcb5316c4793dd4c", "shape": [ 204, 17 ] } ], "region": "r:28", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 33.45360234820312, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:674", "inputs": [ "s:675" ], "name": "broadcast_select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2c93ccfa935dc7b48ae36ca9b98d2c3e3e50962957d9f3b464865f5c129d7072", "shape": [ 204, 216 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "08720f852c879614691b2c33f2980888e7cdf29871f0a4e9e60e7446758bfc2e", "shape": [ 204, 204 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7e26b5245da34ef2eb9e8e57159215ac9a011c4de6a4aa42cdbfb51763b7de8f", "shape": [ 204, 216 ] } ], "region": "r:692", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 204 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0149999999999997))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "s:675", "inputs": [ "s:29", "s:676" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:691", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 216 }, { "annotation": null, "id": "s:676", "inputs": [ "s:598", "s:621", "s:621", "s:621", "s:621", "s:621", "s:568", "s:598", "s:621", "s:621", "s:621", "s:621" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:693", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "s:677", "inputs": [ "s:678" ], "name": "cond_gate", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "15e2148326de0ebea480fee5ab6ac73972e5dd2b77509f9ab2b4f299a2e13d5e", "shape": [ 17, 18 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "9dfa7d580cc225ce9275db41bface936b3be2fea45f33eb9ead196c15c7ee85a", "shape": [ 17, 17 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "a59d363b8093f872f1f5459af5b27535b5c71231d92a1d7e7946f1a0bc0ce20f", "shape": [ 17, 18 ] } ], "region": "r:695", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 17 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=36.49483892531249))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 36.49483892531249, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 36.49483892531249, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:678", "inputs": [ "s:679", "s:685" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:694", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 36.49483892531249, "upper_unbounded": false } }, "width": 18 }, { "annotation": null, "id": "s:679", "inputs": [ "s:680" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "060e78c0da0aaa3589d0508fd400a633facf11bd402f8ad79b26a83d3eee05ca", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:698", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" }, { "annotation": null, "kind": "assert", "message": "cond near \u00b11 (c_tol=0.005)" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:680", "inputs": [ "s:681" ], "name": null, "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "80b8fd6d60fa85fd14a38b5295cb92abd80dfec5ca406c9f969609a79d36809d", "shape": [ 2, 1 ] } ], "region": "r:699", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -2.008702015625, "lower_unbounded": false, "upper": 2.008702015625, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:681", "inputs": [ "s:682", "s:683" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:699", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 2 }, { "annotation": null, "id": "s:682", "inputs": [ "s:18" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "fcc789a616d690b13c0ce2bb32111dec233ff36515f19e597b74cfe3b5615bcd", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:701", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:683", "inputs": [ "s:684" ], "name": "compare", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "ee1c1e99aec431b3878503f437c2d6ed686bab105f28857153577a62843be75e", "shape": [ 2 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "850f0296023889cd036c1ce2fbdeba8994988810fab9e666b3d95dda250e4db7", "shape": [ 2, 1 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "50a0c0693a3ae7e2c27c35f980063462086ee66ef6683f833858cd80a6365d2c", "shape": [ 2, 1 ] } ], "region": "r:703", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": true, "lanes": 2 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=-1.0043510078125, hi=1.0043510078125))" } ], "claimed_range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false }, "integer": false, "range": { "lower": -1.0043510078125, "lower_unbounded": false, "upper": 1.0043510078125, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:684", "inputs": [ "s:18" ], "name": "negate", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", "shape": [ 1 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "c68830a25204a09f8e77aada6bc5807f607cccaaa0ebb2a7122d317584478a8b", "shape": [ 1, 1 ] } ], "region": "r:704", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -9.50850981139363, "lower_unbounded": false, "upper": 0.3418431447269631, "upper_unbounded": false } }, "width": 1 }, { "annotation": null, "id": "s:685", "inputs": [ "s:686" ], "name": "remove_leading_0s_sum", "op": "Linear", "parameters": [ { "dtype": "float32", "name": "output_bias", "sha256": "1751ac12e70e15b4f76c16775cd329ae55973b612521dab2de828a5cdb6c8ab3", "shape": [ 17 ] }, { "dtype": "float32", "name": "output_matrix", "sha256": "d7c028b7eda356619d577d7e3dfef5da2267e2c58f989f7adcb5316c4793dd4c", "shape": [ 204, 17 ] } ], "region": "r:28", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 36.49483892531249, "upper_unbounded": false } }, "width": 17 }, { "annotation": null, "id": "s:686", "inputs": [ "s:687" ], "name": "broadcast_select", "op": "FFN", "parameters": [ { "dtype": "float32", "name": "gate_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "gate_proj", "sha256": "2c93ccfa935dc7b48ae36ca9b98d2c3e3e50962957d9f3b464865f5c129d7072", "shape": [ 204, 216 ] }, { "dtype": "float32", "name": "out_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "out_proj", "sha256": "08720f852c879614691b2c33f2980888e7cdf29871f0a4e9e60e7446758bfc2e", "shape": [ 204, 204 ] }, { "dtype": "float32", "name": "up_bias", "sha256": "0645a4a67dcec462dc9f335bb0564e6e39bf12ea7e40cf8de81418210102c2d1", "shape": [ 204 ] }, { "dtype": "float32", "name": "up_proj", "sha256": "7e26b5245da34ef2eb9e8e57159215ac9a011c4de6a4aa42cdbfb51763b7de8f", "shape": [ 204, 216 ] } ], "region": "r:706", "scheduling_predecessors": [], "semantics": { "activation": "swish", "degenerate_up": false, "lanes": 204 }, "value_contract": { "checks": [ { "annotation": null, "kind": "assert", "message": "matches NodeValueType(value_range=Range(lo=0.0, hi=3.0149999999999997))" } ], "claimed_range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false }, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "s:687", "inputs": [ "s:29", "s:688" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:705", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": -1.008702015625, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 216 }, { "annotation": null, "id": "s:688", "inputs": [ "s:621", "s:621", "s:621", "s:621", "s:621", "s:621", "s:598", "s:621", "s:621", "s:621", "s:621", "s:621" ], "name": null, "op": "Concatenate", "parameters": [], "region": "r:707", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 3.0149999999999997, "upper_unbounded": false } }, "width": 204 }, { "annotation": null, "id": "s:689", "inputs": [], "name": null, "op": "LiteralValue", "parameters": [ { "dtype": "float32", "name": "value", "sha256": "bdc7bf3db562e39b656aed4e81ea8425eddc35662c76ad9ef2f006e36d6c43d0", "shape": [ 17 ], "values": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0 ] } ], "region": "r:708", "scheduling_predecessors": [], "value_contract": { "checks": [], "claimed_range": null, "integer": false, "range": { "lower": 0.0, "lower_unbounded": false, "upper": 1.0, "upper_unbounded": false } }, "width": 17 } ], "root": "s:0", "semantic_regions": [ { "annotation": null, "id": "r:0", "op": "swiglu.sequence_ops.output_sequence", "operands": [ "s:7", "s:26", "s:594", "s:617", "s:637", "s:649", "s:661", "s:673", "s:685" ], "params": { "default_output": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:0" ] }, { "annotation": null, "id": "r:1", "op": "swiglu.map_select.select", "operands": [ "s:2", "s:10", "s:689" ], "params": {}, "parent": "r:0", "results": [ "s:0" ] }, { "annotation": null, "id": "r:2", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:1" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 35 ] } }, "name": "select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 35 ] } } }, "parent": "r:1", "results": [ "s:0" ] }, { "annotation": null, "id": "r:3", "op": "attention_ops.get_prev_value", "operands": [ "s:7", "s:7" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": "r:0", "results": [ "s:2" ] }, { "annotation": null, "id": "r:4", "op": "swiglu.logic_ops.equals_vector", "operands": [ "s:8" ], "params": { "vector": { "tensor": { "dtype": "float32", "shape": [ 17 ] } } }, "parent": null, "results": [ "s:7" ] }, { "annotation": null, "id": "r:5", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:8" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 17 ] } }, "name": "equals_vector", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:4", "results": [ "s:7" ] }, { "annotation": null, "id": "r:6", "op": "inout_nodes.create_onehot_embedding", "operands": [], "params": { "vocab": [ "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "+", "-", "*", "\n", " ", "", "" ] }, "parent": null, "results": [ "s:8" ] }, { "annotation": null, "id": "r:7", "op": "linear.sum_nodes", "operands": [ "s:12", "s:586", "s:609", "s:629", "s:641", "s:653", "s:665", "s:677" ], "params": { "max_fanout": null }, "parent": "r:0", "results": [ "s:10" ] }, { "annotation": null, "id": "r:8", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:14", "s:26" ], "params": {}, "parent": "r:0", "results": [ "s:12" ] }, { "annotation": null, "id": "r:9", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:13" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:8", "results": [ "s:12" ] }, { "annotation": null, "id": "r:10", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:17", "s:24" ], "params": {}, "parent": "r:0", "results": [ "s:14" ] }, { "annotation": null, "id": "r:11", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:15" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:10", "results": [ "s:14" ] }, { "annotation": null, "id": "r:12", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:15" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:11", "results": [ "s:14" ] }, { "annotation": null, "id": "r:13", "op": "linear.sum_nodes", "operands": [ "s:17", "s:24" ], "params": { "max_fanout": null }, "parent": "r:10", "results": [ "s:15" ] }, { "annotation": null, "id": "r:14", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:18" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -0.5, "true_level": 1.0 }, "parent": "r:0", "results": [ "s:17" ] }, { "annotation": null, "id": "r:15", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:18" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:14", "results": [ "s:17" ] }, { "annotation": null, "id": "r:16", "op": "swiglu.marker_count.count_since_marker", "operands": [ "s:2", "s:22" ], "params": { "max_gap": 9, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": "r:0", "results": [ "s:18" ] }, { "annotation": null, "id": "r:17", "op": "linear.add_const", "operands": [ "s:19" ], "params": { "scalar": -1.0 }, "parent": "r:16", "results": [ "s:18" ] }, { "annotation": null, "id": "r:18", "op": "swiglu.arithmetic_ops.reciprocal", "operands": [ "s:20" ], "params": { "max_value": 1.5, "min_value": 0.09523809523809523, "step": 0.045314900153609824 }, "parent": "r:16", "results": [ "s:19" ] }, { "annotation": null, "id": "r:19", "op": "swiglu.arithmetic_ops.piecewise_linear", "operands": [ "s:20" ], "params": { "breakpoints": [ 0.09523809523809523, 0.10409566750659383, 0.11377703393325501, 0.12435881108912687, 0.1359247412318152, 0.14856635502646182, 0.162383695902762, 0.17748611178045046, 0.19399312042882713, 0.21203535530862097, 0.2317555993814113, 0.2533099150679869, 0.27686887929792725, 0.3026189334243631, 0.33076385868689684, 0.36152638889922667, 0.3951499731239911, 0.43190070228432187, 0.472069414958958, 0.5159739990257997, 0.5639619073686907, 0.6164129075562749, 0.6737420872533829, 0.7364031391492301, 0.8048919513987473, 0.8797505319911436, 0.9615712981024407, 1.0510017643771328, 1.1487496672411774, 1.2555885657992032, 1.3723639636404061, 1.5 ], "clamp": true, "d_max": 1024, "fn": "", "input_scale": 29.988465456237808, "name": "reciprocal" }, "parent": "r:18", "results": [ "s:19" ] }, { "annotation": null, "id": "r:20", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:20" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 32 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 32, 1 ] } }, "name": "reciprocal_0_32", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 32, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:19", "results": [ "s:19" ] }, { "annotation": null, "id": "r:21", "op": "attention_ops.attend_mean_where", "operands": [ "s:2", "s:22" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": "r:16", "results": [ "s:20" ] }, { "annotation": null, "id": "r:22", "op": "linear.bool_to_01", "operands": [ "s:7" ], "params": {}, "parent": "r:0", "results": [ "s:22" ] }, { "annotation": null, "id": "r:23", "op": "linear.multiply_const", "operands": [ "s:23" ], "params": { "scalar": 0.5 }, "parent": "r:22", "results": [ "s:22" ] }, { "annotation": null, "id": "r:24", "op": "linear.add_const", "operands": [ "s:7" ], "params": { "scalar": 1.0 }, "parent": "r:22", "results": [ "s:23" ] }, { "annotation": null, "id": "r:25", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:25" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -0.5, "true_level": 1.0 }, "parent": "r:0", "results": [ "s:24" ] }, { "annotation": null, "id": "r:26", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:25" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:25", "results": [ "s:24" ] }, { "annotation": null, "id": "r:27", "op": "linear.negate", "operands": [ "s:18" ], "params": {}, "parent": "r:0", "results": [ "s:25" ] }, { "annotation": null, "id": "r:28", "op": "swiglu.sequence_ops.remove_leading_0s", "operands": [ "s:8", "s:42", "s:422", "s:447", "s:482", "s:542", "s:568", "s:598", "s:621", "s:562" ], "params": { "max_removals": 5, "sign_token": { "tensor": { "dtype": "float32", "shape": [ 17 ] } } }, "parent": null, "results": [ "s:26", "s:594", "s:617", "s:637", "s:649", "s:661", "s:673", "s:685" ] }, { "annotation": null, "id": "r:29", "op": "swiglu.map_select.broadcast_select", "operands": [ "s:29", "s:567" ], "params": { "d_fill": 17, "n_slots": 12 }, "parent": "r:28", "results": [ "s:27" ] }, { "annotation": null, "id": "r:30", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:28" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 204 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 204, 216 ] } }, "name": "broadcast_select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 204 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 204, 204 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 204 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 204, 216 ] } } }, "parent": "r:29", "results": [ "s:27" ] }, { "annotation": null, "id": "r:31", "op": "swiglu.map_select.in_range", "operands": [ "s:31", "s:566" ], "params": { "n_slots": 12 }, "parent": "r:28", "results": [ "s:29" ] }, { "annotation": null, "id": "r:32", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:30" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 48 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 48, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 12 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 48, 12 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:31", "results": [ "s:29" ] }, { "annotation": null, "id": "r:33", "op": "linear.add", "operands": [ "s:32", "s:559" ], "params": {}, "parent": "r:28", "results": [ "s:31" ] }, { "annotation": null, "id": "r:34", "op": "linear.sum_nodes", "operands": [ "s:34", "s:414", "s:439", "s:474", "s:534" ], "params": { "max_fanout": null }, "parent": "r:28", "results": [ "s:32" ] }, { "annotation": null, "id": "r:35", "op": "linear.bool_to_01", "operands": [ "s:36" ], "params": {}, "parent": "r:28", "results": [ "s:34" ] }, { "annotation": null, "id": "r:36", "op": "linear.multiply_const", "operands": [ "s:35" ], "params": { "scalar": 0.5 }, "parent": "r:35", "results": [ "s:34" ] }, { "annotation": null, "id": "r:37", "op": "linear.add_const", "operands": [ "s:36" ], "params": { "scalar": 1.0 }, "parent": "r:35", "results": [ "s:35" ] }, { "annotation": null, "id": "r:38", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:37" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 0.5, "true_level": 1.0 }, "parent": "r:28", "results": [ "s:36" ] }, { "annotation": null, "id": "r:39", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:37" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:38", "results": [ "s:36" ] }, { "annotation": null, "id": "r:40", "op": "linear.sum_nodes", "operands": [ "s:39" ], "params": { "max_fanout": null }, "parent": "r:28", "results": [ "s:37" ] }, { "annotation": null, "id": "r:41", "op": "linear.bool_to_01", "operands": [ "s:41" ], "params": {}, "parent": "r:28", "results": [ "s:39" ] }, { "annotation": null, "id": "r:42", "op": "linear.multiply_const", "operands": [ "s:40" ], "params": { "scalar": 0.5 }, "parent": "r:41", "results": [ "s:39" ] }, { "annotation": null, "id": "r:43", "op": "linear.add_const", "operands": [ "s:41" ], "params": { "scalar": 1.0 }, "parent": "r:41", "results": [ "s:40" ] }, { "annotation": null, "id": "r:44", "op": "swiglu.logic_ops.equals_vector", "operands": [ "s:42" ], "params": { "vector": { "tensor": { "dtype": "float32", "shape": [ 17 ] } } }, "parent": "r:28", "results": [ "s:41" ] }, { "annotation": null, "id": "r:45", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:42" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 17 ] } }, "name": "equals_vector", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:44", "results": [ "s:41" ] }, { "annotation": null, "id": "r:46", "op": "swiglu.map_select.switch", "operands": [ "s:46", "s:74", "s:83", "s:71", "s:80", "s:89" ], "params": {}, "parent": null, "results": [ "s:42" ] }, { "annotation": null, "id": "r:47", "op": "linear.sum_nodes", "operands": [ "s:44", "s:72", "s:81" ], "params": { "max_fanout": null }, "parent": "r:46", "results": [ "s:42" ] }, { "annotation": null, "id": "r:48", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:46", "s:71" ], "params": {}, "parent": "r:46", "results": [ "s:44" ] }, { "annotation": null, "id": "r:49", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:45" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:48", "results": [ "s:44" ] }, { "annotation": null, "id": "r:50", "op": "attention_ops.get_prev_value", "operands": [ "s:58", "s:51" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:46" ] }, { "annotation": null, "id": "r:51", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:54", "s:63" ], "params": {}, "parent": null, "results": [ "s:51" ] }, { "annotation": null, "id": "r:52", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:52" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:51", "results": [ "s:51" ] }, { "annotation": null, "id": "r:53", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:52" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:52", "results": [ "s:51" ] }, { "annotation": null, "id": "r:54", "op": "linear.sum_nodes", "operands": [ "s:54", "s:63" ], "params": { "max_fanout": null }, "parent": "r:51", "results": [ "s:52" ] }, { "annotation": null, "id": "r:55", "op": "swiglu.logic_ops.bool_any_true", "operands": [ "s:58", "s:60", "s:62" ], "params": {}, "parent": null, "results": [ "s:54" ] }, { "annotation": null, "id": "r:56", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:55" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 0.5, "true_level": 1.0 }, "parent": "r:55", "results": [ "s:54" ] }, { "annotation": null, "id": "r:57", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:55" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:56", "results": [ "s:54" ] }, { "annotation": null, "id": "r:58", "op": "linear.sum_nodes", "operands": [ "s:57", "s:59", "s:61" ], "params": { "max_fanout": null }, "parent": "r:55", "results": [ "s:55" ] }, { "annotation": null, "id": "r:59", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:58" ], "params": { "false_level": 0.0, "sharpness": null, "thresh": 0.0, "true_level": 1.0 }, "parent": "r:55", "results": [ "s:57" ] }, { "annotation": null, "id": "r:60", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:58" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:59", "results": [ "s:57" ] }, { "annotation": null, "id": "r:61", "op": "swiglu.logic_ops.equals_vector", "operands": [ "s:8" ], "params": { "vector": { "tensor": { "dtype": "float32", "shape": [ 17 ] } } }, "parent": null, "results": [ "s:58" ] }, { "annotation": null, "id": "r:62", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:8" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 17 ] } }, "name": "equals_vector", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:61", "results": [ "s:58" ] }, { "annotation": null, "id": "r:63", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:60" ], "params": { "false_level": 0.0, "sharpness": null, "thresh": 0.0, "true_level": 1.0 }, "parent": "r:55", "results": [ "s:59" ] }, { "annotation": null, "id": "r:64", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:60" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:63", "results": [ "s:59" ] }, { "annotation": null, "id": "r:65", "op": "swiglu.logic_ops.equals_vector", "operands": [ "s:8" ], "params": { "vector": { "tensor": { "dtype": "float32", "shape": [ 17 ] } } }, "parent": null, "results": [ "s:60" ] }, { "annotation": null, "id": "r:66", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:8" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 17 ] } }, "name": "equals_vector", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:65", "results": [ "s:60" ] }, { "annotation": null, "id": "r:67", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:62" ], "params": { "false_level": 0.0, "sharpness": null, "thresh": 0.0, "true_level": 1.0 }, "parent": "r:55", "results": [ "s:61" ] }, { "annotation": null, "id": "r:68", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:62" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:67", "results": [ "s:61" ] }, { "annotation": null, "id": "r:69", "op": "swiglu.logic_ops.equals_vector", "operands": [ "s:8" ], "params": { "vector": { "tensor": { "dtype": "float32", "shape": [ 17 ] } } }, "parent": null, "results": [ "s:62" ] }, { "annotation": null, "id": "r:70", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:8" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 17 ] } }, "name": "equals_vector", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:69", "results": [ "s:62" ] }, { "annotation": null, "id": "r:71", "op": "swiglu.logic_ops.bool_not", "operands": [ "s:64" ], "params": {}, "parent": null, "results": [ "s:63" ] }, { "annotation": null, "id": "r:72", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:64" ], "params": { "false_level": 1.0, "sharpness": null, "thresh": 0.0, "true_level": -1.0 }, "parent": "r:71", "results": [ "s:63" ] }, { "annotation": null, "id": "r:73", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:64" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:72", "results": [ "s:63" ] }, { "annotation": null, "id": "r:74", "op": "attention_ops.get_prev_value", "operands": [ "s:7", "s:7" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:64" ] }, { "annotation": null, "id": "r:75", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 17 ] } } }, "parent": null, "results": [ "s:71" ] }, { "annotation": null, "id": "r:76", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:74", "s:80" ], "params": {}, "parent": "r:46", "results": [ "s:72" ] }, { "annotation": null, "id": "r:77", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:73" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:76", "results": [ "s:72" ] }, { "annotation": null, "id": "r:78", "op": "attention_ops.get_prev_value", "operands": [ "s:60", "s:51" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:74" ] }, { "annotation": null, "id": "r:79", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 17 ] } } }, "parent": null, "results": [ "s:80" ] }, { "annotation": null, "id": "r:80", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:83", "s:89" ], "params": {}, "parent": "r:46", "results": [ "s:81" ] }, { "annotation": null, "id": "r:81", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:82" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:80", "results": [ "s:81" ] }, { "annotation": null, "id": "r:82", "op": "attention_ops.get_prev_value", "operands": [ "s:62", "s:51" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:83" ] }, { "annotation": null, "id": "r:83", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:90" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:89" ] }, { "annotation": null, "id": "r:84", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:90" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 200 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 36 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 17 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:83", "results": [ "s:89" ] }, { "annotation": null, "id": "r:85", "op": "linear.concat", "operands": [ "s:91", "s:228", "s:280" ], "params": {}, "parent": null, "results": [ "s:90" ] }, { "annotation": null, "id": "r:86", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:92" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:91" ] }, { "annotation": null, "id": "r:87", "op": "linear.bool_to_01", "operands": [ "s:94" ], "params": {}, "parent": null, "results": [ "s:92" ] }, { "annotation": null, "id": "r:88", "op": "linear.multiply_const", "operands": [ "s:93" ], "params": { "scalar": 0.5 }, "parent": "r:87", "results": [ "s:92" ] }, { "annotation": null, "id": "r:89", "op": "linear.add_const", "operands": [ "s:94" ], "params": { "scalar": 1.0 }, "parent": "r:87", "results": [ "s:93" ] }, { "annotation": null, "id": "r:90", "op": "swiglu.map_select.in_range", "operands": [ "s:96", "s:227" ], "params": { "n_slots": 100 }, "parent": null, "results": [ "s:94" ] }, { "annotation": null, "id": "r:91", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:95" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 400 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 400, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 100 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 400, 100 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:90", "results": [ "s:94" ] }, { "annotation": null, "id": "r:92", "op": "linear.sum_nodes", "operands": [ "s:98", "s:222", "s:224", "s:225", "s:226" ], "params": { "max_fanout": null }, "parent": null, "results": [ "s:96" ] }, { "annotation": null, "id": "r:93", "op": "linear.slice_columns", "operands": [ "s:100" ], "params": { "name": "pp_tens", "start": 0, "width": 17 }, "parent": null, "results": [ "s:99" ] }, { "annotation": null, "id": "r:94", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:101" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:100" ] }, { "annotation": null, "id": "r:95", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:101" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 100 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 34 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 34 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:94", "results": [ "s:100" ] }, { "annotation": null, "id": "r:96", "op": "linear.concat", "operands": [ "s:102", "s:166" ], "params": {}, "parent": null, "results": [ "s:101" ] }, { "annotation": null, "id": "r:97", "op": "attention_ops.get_prev_value", "operands": [ "s:108", "s:7" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:102" ] }, { "annotation": null, "id": "r:98", "op": "swiglu.sequence_ops.IndexedRegion.token_at", "operands": [ "s:114" ], "params": {}, "parent": null, "results": [ "s:108" ] }, { "annotation": null, "id": "r:99", "op": "attention_ops.attend_argmax_dot", "operands": [ "s:109", "s:138", "s:163" ], "params": { "assert_hardness_gt": null, "match_gain": 2000000.0, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": "r:98", "results": [ "s:108" ] }, { "annotation": null, "id": "r:100", "op": "linear.concat", "operands": [ "s:110", "s:136" ], "params": {}, "parent": "r:98", "results": [ "s:109" ] }, { "annotation": null, "id": "r:101", "op": "linear.bool_to_01", "operands": [ "s:112" ], "params": {}, "parent": "r:98", "results": [ "s:110" ] }, { "annotation": null, "id": "r:102", "op": "linear.multiply_const", "operands": [ "s:111" ], "params": { "scalar": 0.5 }, "parent": "r:101", "results": [ "s:110" ] }, { "annotation": null, "id": "r:103", "op": "linear.add_const", "operands": [ "s:112" ], "params": { "scalar": 1.0 }, "parent": "r:101", "results": [ "s:111" ] }, { "annotation": null, "id": "r:104", "op": "swiglu.map_select.in_range", "operands": [ "s:114", "s:135" ], "params": { "n_slots": 3 }, "parent": "r:98", "results": [ "s:112" ] }, { "annotation": null, "id": "r:105", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:113" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 12 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:104", "results": [ "s:112" ] }, { "annotation": null, "id": "r:106", "op": "linear.add_const", "operands": [ "s:115" ], "params": { "scalar": -3.0 }, "parent": null, "results": [ "s:114" ] }, { "annotation": null, "id": "r:107", "op": "swiglu.sequence_ops.IndexedRegion.length_at", "operands": [ "s:51" ], "params": {}, "parent": null, "results": [ "s:115" ] }, { "annotation": null, "id": "r:108", "op": "linear.add_const", "operands": [ "s:116" ], "params": { "scalar": -1.0 }, "parent": "r:107", "results": [ "s:115" ] }, { "annotation": null, "id": "r:109", "op": "attention_ops.get_prev_value", "operands": [ "s:122", "s:51" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": "r:107", "results": [ "s:116" ] }, { "annotation": null, "id": "r:110", "op": "swiglu.sequence_ops.IndexedRegion.__init__", "operands": [ "s:8", "s:131", "s:142" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "max_len": 3, "max_read_distance": 10, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [] }, { "annotation": null, "id": "r:111", "op": "swiglu.marker_count.count_since_marker", "operands": [ "s:126", "s:133" ], "params": { "max_gap": 4, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": "r:110", "results": [ "s:122" ] }, { "annotation": null, "id": "r:112", "op": "linear.add_const", "operands": [ "s:123" ], "params": { "scalar": -1.0 }, "parent": "r:111", "results": [ "s:122" ] }, { "annotation": null, "id": "r:113", "op": "swiglu.arithmetic_ops.reciprocal", "operands": [ "s:124" ], "params": { "max_value": 1.5, "min_value": 0.18181818181818182, "step": 0.04252199413489736 }, "parent": "r:111", "results": [ "s:123" ] }, { "annotation": null, "id": "r:114", "op": "swiglu.arithmetic_ops.piecewise_linear", "operands": [ "s:124" ], "params": { "breakpoints": [ 0.18181818181818182, 0.19462576879929028, 0.20833554434393137, 0.2230110601738595, 0.23872034470395204, 0.2555362183882104, 0.2735366312792783, 0.2928050243662364, 0.3134307163656553, 0.33550931775887877, 0.35914317399480933, 0.3844418399126637, 0.4115225875838872, 0.44051094992733025, 0.4715413026176177, 0.5047574869841495, 0.5403134767881832, 0.5783740919688469, 0.6191157626666555, 0.6627273470661621, 0.7094110068488624, 0.7593831443145113, 0.8128754055148878, 0.8701357540500343, 0.9314296205045596, 0.997041132852221, 1.0672744335323334, 1.142455089303316, 1.2229316004087583, 1.3090770160517557, 1.4012906636660458, 1.5 ], "clamp": true, "d_max": 1024, "fn": "", "input_scale": 20.739660046174524, "name": "reciprocal" }, "parent": "r:113", "results": [ "s:123" ] }, { "annotation": null, "id": "r:115", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:124" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 32 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 32, 1 ] } }, "name": "reciprocal_0_32", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 32, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:114", "results": [ "s:123" ] }, { "annotation": null, "id": "r:116", "op": "attention_ops.attend_mean_where", "operands": [ "s:126", "s:133" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": "r:111", "results": [ "s:124" ] }, { "annotation": null, "id": "r:117", "op": "attention_ops.get_prev_value", "operands": [ "s:131", "s:131" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": "r:110", "results": [ "s:126" ] }, { "annotation": null, "id": "r:118", "op": "swiglu.logic_ops.equals_vector", "operands": [ "s:8" ], "params": { "vector": { "tensor": { "dtype": "float32", "shape": [ 17 ] } } }, "parent": null, "results": [ "s:131" ] }, { "annotation": null, "id": "r:119", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:8" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 17 ] } }, "name": "equals_vector", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:118", "results": [ "s:131" ] }, { "annotation": null, "id": "r:120", "op": "linear.bool_to_01", "operands": [ "s:131" ], "params": {}, "parent": "r:110", "results": [ "s:133" ] }, { "annotation": null, "id": "r:121", "op": "linear.multiply_const", "operands": [ "s:134" ], "params": { "scalar": 0.5 }, "parent": "r:120", "results": [ "s:133" ] }, { "annotation": null, "id": "r:122", "op": "linear.add_const", "operands": [ "s:131" ], "params": { "scalar": 1.0 }, "parent": "r:120", "results": [ "s:134" ] }, { "annotation": null, "id": "r:123", "op": "linear.add_const", "operands": [ "s:114" ], "params": { "scalar": 1.0 }, "parent": "r:98", "results": [ "s:135" ] }, { "annotation": null, "id": "r:124", "op": "linear.add_const", "operands": [ "s:137" ], "params": { "scalar": 1.0 }, "parent": "r:98", "results": [ "s:136" ] }, { "annotation": null, "id": "r:125", "op": "linear.add", "operands": [ "s:139", "s:160" ], "params": {}, "parent": "r:110", "results": [ "s:138" ] }, { "annotation": null, "id": "r:126", "op": "linear.concat", "operands": [ "s:140", "s:159" ], "params": {}, "parent": "r:110", "results": [ "s:139" ] }, { "annotation": null, "id": "r:127", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:142", "s:153" ], "params": {}, "parent": "r:110", "results": [ "s:140" ] }, { "annotation": null, "id": "r:128", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:141" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 3, 4 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 3, 3 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 3, 4 ] } } }, "parent": "r:127", "results": [ "s:140" ] }, { "annotation": null, "id": "r:129", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:145", "s:146" ], "params": {}, "parent": null, "results": [ "s:142" ] }, { "annotation": null, "id": "r:130", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:143" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:129", "results": [ "s:142" ] }, { "annotation": null, "id": "r:131", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:143" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:130", "results": [ "s:142" ] }, { "annotation": null, "id": "r:132", "op": "linear.sum_nodes", "operands": [ "s:145", "s:146" ], "params": { "max_fanout": null }, "parent": "r:129", "results": [ "s:143" ] }, { "annotation": null, "id": "r:133", "op": "swiglu.sequence_ops.check_is_digit", "operands": [ "s:8" ], "params": {}, "parent": null, "results": [ "s:145" ] }, { "annotation": null, "id": "r:134", "op": "swiglu.map_select.map_to_table", "operands": [ "s:8" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": "r:133", "results": [ "s:145" ] }, { "annotation": null, "id": "r:135", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:8" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 10 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 10, 17 ] } }, "name": "map_to_table", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 10, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:134", "results": [ "s:145" ] }, { "annotation": null, "id": "r:136", "op": "swiglu.logic_ops.bool_not", "operands": [ "s:147" ], "params": {}, "parent": null, "results": [ "s:146" ] }, { "annotation": null, "id": "r:137", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:147" ], "params": { "false_level": 1.0, "sharpness": null, "thresh": 0.0, "true_level": -1.0 }, "parent": "r:136", "results": [ "s:146" ] }, { "annotation": null, "id": "r:138", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:147" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:137", "results": [ "s:146" ] }, { "annotation": null, "id": "r:139", "op": "attention_ops.get_prev_value", "operands": [ "s:51", "s:51" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:147" ] }, { "annotation": null, "id": "r:140", "op": "linear.bool_to_01", "operands": [ "s:155" ], "params": {}, "parent": "r:110", "results": [ "s:153" ] }, { "annotation": null, "id": "r:141", "op": "linear.multiply_const", "operands": [ "s:154" ], "params": { "scalar": 0.5 }, "parent": "r:140", "results": [ "s:153" ] }, { "annotation": null, "id": "r:142", "op": "linear.add_const", "operands": [ "s:155" ], "params": { "scalar": 1.0 }, "parent": "r:140", "results": [ "s:154" ] }, { "annotation": null, "id": "r:143", "op": "swiglu.map_select.in_range", "operands": [ "s:157", "s:158" ], "params": { "n_slots": 3 }, "parent": "r:110", "results": [ "s:155" ] }, { "annotation": null, "id": "r:144", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:156" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 12 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:143", "results": [ "s:155" ] }, { "annotation": null, "id": "r:145", "op": "linear.add_const", "operands": [ "s:122" ], "params": { "scalar": -1.0 }, "parent": "r:110", "results": [ "s:157" ] }, { "annotation": null, "id": "r:146", "op": "linear.add_const", "operands": [ "s:157" ], "params": { "scalar": 1.0 }, "parent": "r:110", "results": [ "s:158" ] }, { "annotation": null, "id": "r:147", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 1 ] } } }, "parent": "r:110", "results": [ "s:159" ] }, { "annotation": null, "id": "r:148", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:131", "s:162" ], "params": {}, "parent": "r:110", "results": [ "s:160" ] }, { "annotation": null, "id": "r:149", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:161" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 4 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 4, 5 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 4 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 4, 4 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 4 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 4, 5 ] } } }, "parent": "r:148", "results": [ "s:160" ] }, { "annotation": null, "id": "r:150", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 4 ] } } }, "parent": "r:110", "results": [ "s:162" ] }, { "annotation": null, "id": "r:151", "op": "swiglu.map_select.select", "operands": [ "s:142", "s:8", "s:165" ], "params": {}, "parent": "r:110", "results": [ "s:163" ] }, { "annotation": null, "id": "r:152", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:164" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 35 ] } }, "name": "select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 35 ] } } }, "parent": "r:151", "results": [ "s:163" ] }, { "annotation": null, "id": "r:153", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 17 ] } } }, "parent": "r:110", "results": [ "s:165" ] }, { "annotation": null, "id": "r:154", "op": "attention_ops.get_prev_value", "operands": [ "s:172", "s:7" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:166" ] }, { "annotation": null, "id": "r:155", "op": "swiglu.sequence_ops.IndexedRegion.token_at", "operands": [ "s:178" ], "params": {}, "parent": null, "results": [ "s:172" ] }, { "annotation": null, "id": "r:156", "op": "attention_ops.attend_argmax_dot", "operands": [ "s:173", "s:201", "s:219" ], "params": { "assert_hardness_gt": null, "match_gain": 2000000.0, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": "r:155", "results": [ "s:172" ] }, { "annotation": null, "id": "r:157", "op": "linear.concat", "operands": [ "s:174", "s:199" ], "params": {}, "parent": "r:155", "results": [ "s:173" ] }, { "annotation": null, "id": "r:158", "op": "linear.bool_to_01", "operands": [ "s:176" ], "params": {}, "parent": "r:155", "results": [ "s:174" ] }, { "annotation": null, "id": "r:159", "op": "linear.multiply_const", "operands": [ "s:175" ], "params": { "scalar": 0.5 }, "parent": "r:158", "results": [ "s:174" ] }, { "annotation": null, "id": "r:160", "op": "linear.add_const", "operands": [ "s:176" ], "params": { "scalar": 1.0 }, "parent": "r:158", "results": [ "s:175" ] }, { "annotation": null, "id": "r:161", "op": "swiglu.map_select.in_range", "operands": [ "s:178", "s:198" ], "params": { "n_slots": 3 }, "parent": "r:155", "results": [ "s:176" ] }, { "annotation": null, "id": "r:162", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:177" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 12 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:161", "results": [ "s:176" ] }, { "annotation": null, "id": "r:163", "op": "linear.add_const", "operands": [ "s:179" ], "params": { "scalar": -3.0 }, "parent": null, "results": [ "s:178" ] }, { "annotation": null, "id": "r:164", "op": "swiglu.sequence_ops.IndexedRegion.length_at", "operands": [ "s:7" ], "params": {}, "parent": null, "results": [ "s:179" ] }, { "annotation": null, "id": "r:165", "op": "linear.add_const", "operands": [ "s:180" ], "params": { "scalar": -1.0 }, "parent": "r:164", "results": [ "s:179" ] }, { "annotation": null, "id": "r:166", "op": "attention_ops.get_prev_value", "operands": [ "s:186", "s:7" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": "r:164", "results": [ "s:180" ] }, { "annotation": null, "id": "r:167", "op": "swiglu.sequence_ops.IndexedRegion.__init__", "operands": [ "s:8", "s:51", "s:205" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "max_len": 3, "max_read_distance": 10, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [] }, { "annotation": null, "id": "r:168", "op": "swiglu.marker_count.count_since_marker", "operands": [ "s:190", "s:196" ], "params": { "max_gap": 4, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": "r:167", "results": [ "s:186" ] }, { "annotation": null, "id": "r:169", "op": "linear.add_const", "operands": [ "s:187" ], "params": { "scalar": -1.0 }, "parent": "r:168", "results": [ "s:186" ] }, { "annotation": null, "id": "r:170", "op": "swiglu.arithmetic_ops.reciprocal", "operands": [ "s:188" ], "params": { "max_value": 1.5, "min_value": 0.18181818181818182, "step": 0.04252199413489736 }, "parent": "r:168", "results": [ "s:187" ] }, { "annotation": null, "id": "r:171", "op": "swiglu.arithmetic_ops.piecewise_linear", "operands": [ "s:188" ], "params": { "breakpoints": [ 0.18181818181818182, 0.19462576879929028, 0.20833554434393137, 0.2230110601738595, 0.23872034470395204, 0.2555362183882104, 0.2735366312792783, 0.2928050243662364, 0.3134307163656553, 0.33550931775887877, 0.35914317399480933, 0.3844418399126637, 0.4115225875838872, 0.44051094992733025, 0.4715413026176177, 0.5047574869841495, 0.5403134767881832, 0.5783740919688469, 0.6191157626666555, 0.6627273470661621, 0.7094110068488624, 0.7593831443145113, 0.8128754055148878, 0.8701357540500343, 0.9314296205045596, 0.997041132852221, 1.0672744335323334, 1.142455089303316, 1.2229316004087583, 1.3090770160517557, 1.4012906636660458, 1.5 ], "clamp": true, "d_max": 1024, "fn": "", "input_scale": 20.739660046174524, "name": "reciprocal" }, "parent": "r:170", "results": [ "s:187" ] }, { "annotation": null, "id": "r:172", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:188" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 32 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 32, 1 ] } }, "name": "reciprocal_0_32", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 32, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:171", "results": [ "s:187" ] }, { "annotation": null, "id": "r:173", "op": "attention_ops.attend_mean_where", "operands": [ "s:190", "s:196" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": "r:168", "results": [ "s:188" ] }, { "annotation": null, "id": "r:174", "op": "attention_ops.get_prev_value", "operands": [ "s:51", "s:51" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": "r:167", "results": [ "s:190" ] }, { "annotation": null, "id": "r:175", "op": "linear.bool_to_01", "operands": [ "s:51" ], "params": {}, "parent": "r:167", "results": [ "s:196" ] }, { "annotation": null, "id": "r:176", "op": "linear.multiply_const", "operands": [ "s:197" ], "params": { "scalar": 0.5 }, "parent": "r:175", "results": [ "s:196" ] }, { "annotation": null, "id": "r:177", "op": "linear.add_const", "operands": [ "s:51" ], "params": { "scalar": 1.0 }, "parent": "r:175", "results": [ "s:197" ] }, { "annotation": null, "id": "r:178", "op": "linear.add_const", "operands": [ "s:178" ], "params": { "scalar": 1.0 }, "parent": "r:155", "results": [ "s:198" ] }, { "annotation": null, "id": "r:179", "op": "linear.add_const", "operands": [ "s:200" ], "params": { "scalar": 1.0 }, "parent": "r:155", "results": [ "s:199" ] }, { "annotation": null, "id": "r:180", "op": "linear.add", "operands": [ "s:202", "s:216" ], "params": {}, "parent": "r:167", "results": [ "s:201" ] }, { "annotation": null, "id": "r:181", "op": "linear.concat", "operands": [ "s:203", "s:215" ], "params": {}, "parent": "r:167", "results": [ "s:202" ] }, { "annotation": null, "id": "r:182", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:205", "s:209" ], "params": {}, "parent": "r:167", "results": [ "s:203" ] }, { "annotation": null, "id": "r:183", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:204" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 3, 4 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 3, 3 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 3, 4 ] } } }, "parent": "r:182", "results": [ "s:203" ] }, { "annotation": null, "id": "r:184", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:145", "s:147", "s:208" ], "params": {}, "parent": null, "results": [ "s:205" ] }, { "annotation": null, "id": "r:185", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:206" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 2.0, "true_level": 1.0 }, "parent": "r:184", "results": [ "s:205" ] }, { "annotation": null, "id": "r:186", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:206" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:185", "results": [ "s:205" ] }, { "annotation": null, "id": "r:187", "op": "linear.sum_nodes", "operands": [ "s:145", "s:147", "s:208" ], "params": { "max_fanout": null }, "parent": "r:184", "results": [ "s:206" ] }, { "annotation": null, "id": "r:188", "op": "swiglu.logic_ops.bool_not", "operands": [ "s:64" ], "params": {}, "parent": null, "results": [ "s:208" ] }, { "annotation": null, "id": "r:189", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:64" ], "params": { "false_level": 1.0, "sharpness": null, "thresh": 0.0, "true_level": -1.0 }, "parent": "r:188", "results": [ "s:208" ] }, { "annotation": null, "id": "r:190", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:64" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:189", "results": [ "s:208" ] }, { "annotation": null, "id": "r:191", "op": "linear.bool_to_01", "operands": [ "s:211" ], "params": {}, "parent": "r:167", "results": [ "s:209" ] }, { "annotation": null, "id": "r:192", "op": "linear.multiply_const", "operands": [ "s:210" ], "params": { "scalar": 0.5 }, "parent": "r:191", "results": [ "s:209" ] }, { "annotation": null, "id": "r:193", "op": "linear.add_const", "operands": [ "s:211" ], "params": { "scalar": 1.0 }, "parent": "r:191", "results": [ "s:210" ] }, { "annotation": null, "id": "r:194", "op": "swiglu.map_select.in_range", "operands": [ "s:213", "s:214" ], "params": { "n_slots": 3 }, "parent": "r:167", "results": [ "s:211" ] }, { "annotation": null, "id": "r:195", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:212" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 12 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:194", "results": [ "s:211" ] }, { "annotation": null, "id": "r:196", "op": "linear.add_const", "operands": [ "s:186" ], "params": { "scalar": -1.0 }, "parent": "r:167", "results": [ "s:213" ] }, { "annotation": null, "id": "r:197", "op": "linear.add_const", "operands": [ "s:213" ], "params": { "scalar": 1.0 }, "parent": "r:167", "results": [ "s:214" ] }, { "annotation": null, "id": "r:198", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 1 ] } } }, "parent": "r:167", "results": [ "s:215" ] }, { "annotation": null, "id": "r:199", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:51", "s:218" ], "params": {}, "parent": "r:167", "results": [ "s:216" ] }, { "annotation": null, "id": "r:200", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:217" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 4 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 4, 5 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 4 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 4, 4 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 4 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 4, 5 ] } } }, "parent": "r:199", "results": [ "s:216" ] }, { "annotation": null, "id": "r:201", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 4 ] } } }, "parent": "r:167", "results": [ "s:218" ] }, { "annotation": null, "id": "r:202", "op": "swiglu.map_select.select", "operands": [ "s:205", "s:8", "s:221" ], "params": {}, "parent": "r:167", "results": [ "s:219" ] }, { "annotation": null, "id": "r:203", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:220" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 35 ] } }, "name": "select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 35 ] } } }, "parent": "r:202", "results": [ "s:219" ] }, { "annotation": null, "id": "r:204", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 17 ] } } }, "parent": "r:167", "results": [ "s:221" ] }, { "annotation": null, "id": "r:205", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 17 ] } } }, "parent": null, "results": [ "s:223" ] }, { "annotation": null, "id": "r:206", "op": "linear.add_const", "operands": [ "s:96" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:227" ] }, { "annotation": null, "id": "r:207", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:229" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:228" ] }, { "annotation": null, "id": "r:208", "op": "linear.bool_to_01", "operands": [ "s:231" ], "params": {}, "parent": null, "results": [ "s:229" ] }, { "annotation": null, "id": "r:209", "op": "linear.multiply_const", "operands": [ "s:230" ], "params": { "scalar": 0.5 }, "parent": "r:208", "results": [ "s:229" ] }, { "annotation": null, "id": "r:210", "op": "linear.add_const", "operands": [ "s:231" ], "params": { "scalar": 1.0 }, "parent": "r:208", "results": [ "s:230" ] }, { "annotation": null, "id": "r:211", "op": "swiglu.map_select.in_range", "operands": [ "s:233", "s:279" ], "params": { "n_slots": 100 }, "parent": null, "results": [ "s:231" ] }, { "annotation": null, "id": "r:212", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:232" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 400 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 400, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 100 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 400, 100 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:211", "results": [ "s:231" ] }, { "annotation": null, "id": "r:213", "op": "linear.sum_nodes", "operands": [ "s:235", "s:237", "s:257", "s:277", "s:278" ], "params": { "max_fanout": null }, "parent": null, "results": [ "s:233" ] }, { "annotation": null, "id": "r:214", "op": "linear.slice_columns", "operands": [ "s:100" ], "params": { "name": "pp_ones", "start": 17, "width": 17 }, "parent": null, "results": [ "s:236" ] }, { "annotation": null, "id": "r:215", "op": "linear.slice_columns", "operands": [ "s:239" ], "params": { "name": "pp_tens", "start": 0, "width": 17 }, "parent": null, "results": [ "s:238" ] }, { "annotation": null, "id": "r:216", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:240" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:239" ] }, { "annotation": null, "id": "r:217", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:240" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 100 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 34 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 34 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:216", "results": [ "s:239" ] }, { "annotation": null, "id": "r:218", "op": "linear.concat", "operands": [ "s:102", "s:241" ], "params": {}, "parent": null, "results": [ "s:240" ] }, { "annotation": null, "id": "r:219", "op": "attention_ops.get_prev_value", "operands": [ "s:247", "s:7" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:241" ] }, { "annotation": null, "id": "r:220", "op": "swiglu.sequence_ops.IndexedRegion.token_at", "operands": [ "s:253" ], "params": {}, "parent": null, "results": [ "s:247" ] }, { "annotation": null, "id": "r:221", "op": "attention_ops.attend_argmax_dot", "operands": [ "s:248", "s:201", "s:219" ], "params": { "assert_hardness_gt": null, "match_gain": 2000000.0, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": "r:220", "results": [ "s:247" ] }, { "annotation": null, "id": "r:222", "op": "linear.concat", "operands": [ "s:249", "s:255" ], "params": {}, "parent": "r:220", "results": [ "s:248" ] }, { "annotation": null, "id": "r:223", "op": "linear.bool_to_01", "operands": [ "s:251" ], "params": {}, "parent": "r:220", "results": [ "s:249" ] }, { "annotation": null, "id": "r:224", "op": "linear.multiply_const", "operands": [ "s:250" ], "params": { "scalar": 0.5 }, "parent": "r:223", "results": [ "s:249" ] }, { "annotation": null, "id": "r:225", "op": "linear.add_const", "operands": [ "s:251" ], "params": { "scalar": 1.0 }, "parent": "r:223", "results": [ "s:250" ] }, { "annotation": null, "id": "r:226", "op": "swiglu.map_select.in_range", "operands": [ "s:253", "s:254" ], "params": { "n_slots": 3 }, "parent": "r:220", "results": [ "s:251" ] }, { "annotation": null, "id": "r:227", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:252" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 12 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:226", "results": [ "s:251" ] }, { "annotation": null, "id": "r:228", "op": "linear.add_const", "operands": [ "s:179" ], "params": { "scalar": -2.0 }, "parent": null, "results": [ "s:253" ] }, { "annotation": null, "id": "r:229", "op": "linear.add_const", "operands": [ "s:253" ], "params": { "scalar": 1.0 }, "parent": "r:220", "results": [ "s:254" ] }, { "annotation": null, "id": "r:230", "op": "linear.add_const", "operands": [ "s:256" ], "params": { "scalar": 1.0 }, "parent": "r:220", "results": [ "s:255" ] }, { "annotation": null, "id": "r:231", "op": "linear.slice_columns", "operands": [ "s:259" ], "params": { "name": "pp_tens", "start": 0, "width": 17 }, "parent": null, "results": [ "s:258" ] }, { "annotation": null, "id": "r:232", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:260" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:259" ] }, { "annotation": null, "id": "r:233", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:260" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 100 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 34 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 34 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:232", "results": [ "s:259" ] }, { "annotation": null, "id": "r:234", "op": "linear.concat", "operands": [ "s:261", "s:166" ], "params": {}, "parent": null, "results": [ "s:260" ] }, { "annotation": null, "id": "r:235", "op": "attention_ops.get_prev_value", "operands": [ "s:267", "s:7" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:261" ] }, { "annotation": null, "id": "r:236", "op": "swiglu.sequence_ops.IndexedRegion.token_at", "operands": [ "s:273" ], "params": {}, "parent": null, "results": [ "s:267" ] }, { "annotation": null, "id": "r:237", "op": "attention_ops.attend_argmax_dot", "operands": [ "s:268", "s:138", "s:163" ], "params": { "assert_hardness_gt": null, "match_gain": 2000000.0, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": "r:236", "results": [ "s:267" ] }, { "annotation": null, "id": "r:238", "op": "linear.concat", "operands": [ "s:269", "s:275" ], "params": {}, "parent": "r:236", "results": [ "s:268" ] }, { "annotation": null, "id": "r:239", "op": "linear.bool_to_01", "operands": [ "s:271" ], "params": {}, "parent": "r:236", "results": [ "s:269" ] }, { "annotation": null, "id": "r:240", "op": "linear.multiply_const", "operands": [ "s:270" ], "params": { "scalar": 0.5 }, "parent": "r:239", "results": [ "s:269" ] }, { "annotation": null, "id": "r:241", "op": "linear.add_const", "operands": [ "s:271" ], "params": { "scalar": 1.0 }, "parent": "r:239", "results": [ "s:270" ] }, { "annotation": null, "id": "r:242", "op": "swiglu.map_select.in_range", "operands": [ "s:273", "s:274" ], "params": { "n_slots": 3 }, "parent": "r:236", "results": [ "s:271" ] }, { "annotation": null, "id": "r:243", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:272" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 12 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:242", "results": [ "s:271" ] }, { "annotation": null, "id": "r:244", "op": "linear.add_const", "operands": [ "s:115" ], "params": { "scalar": -2.0 }, "parent": null, "results": [ "s:273" ] }, { "annotation": null, "id": "r:245", "op": "linear.add_const", "operands": [ "s:273" ], "params": { "scalar": 1.0 }, "parent": "r:236", "results": [ "s:274" ] }, { "annotation": null, "id": "r:246", "op": "linear.add_const", "operands": [ "s:276" ], "params": { "scalar": 1.0 }, "parent": "r:236", "results": [ "s:275" ] }, { "annotation": null, "id": "r:247", "op": "linear.add_const", "operands": [ "s:233" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:279" ] }, { "annotation": null, "id": "r:248", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:281" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:280" ] }, { "annotation": null, "id": "r:249", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:282" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:281" ] }, { "annotation": null, "id": "r:250", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:282" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 9 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 9, 6 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 9, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:249", "results": [ "s:281" ] }, { "annotation": null, "id": "r:251", "op": "linear.concat", "operands": [ "s:283", "s:333" ], "params": {}, "parent": null, "results": [ "s:282" ] }, { "annotation": null, "id": "r:252", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:284" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:283" ] }, { "annotation": null, "id": "r:253", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:284" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 100 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 34 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:252", "results": [ "s:283" ] }, { "annotation": null, "id": "r:254", "op": "linear.concat", "operands": [ "s:285", "s:223" ], "params": {}, "parent": null, "results": [ "s:284" ] }, { "annotation": null, "id": "r:255", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:286" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:285" ] }, { "annotation": null, "id": "r:256", "op": "linear.bool_to_01", "operands": [ "s:288" ], "params": {}, "parent": null, "results": [ "s:286" ] }, { "annotation": null, "id": "r:257", "op": "linear.multiply_const", "operands": [ "s:287" ], "params": { "scalar": 0.5 }, "parent": "r:256", "results": [ "s:286" ] }, { "annotation": null, "id": "r:258", "op": "linear.add_const", "operands": [ "s:288" ], "params": { "scalar": 1.0 }, "parent": "r:256", "results": [ "s:287" ] }, { "annotation": null, "id": "r:259", "op": "swiglu.map_select.in_range", "operands": [ "s:290", "s:332" ], "params": { "n_slots": 100 }, "parent": null, "results": [ "s:288" ] }, { "annotation": null, "id": "r:260", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:289" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 400 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 400, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 100 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 400, 100 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:259", "results": [ "s:288" ] }, { "annotation": null, "id": "r:261", "op": "linear.sum_nodes", "operands": [ "s:292", "s:328", "s:329", "s:330", "s:331" ], "params": { "max_fanout": null }, "parent": null, "results": [ "s:290" ] }, { "annotation": null, "id": "r:262", "op": "linear.slice_columns", "operands": [ "s:294" ], "params": { "name": "pp_ones", "start": 17, "width": 17 }, "parent": null, "results": [ "s:293" ] }, { "annotation": null, "id": "r:263", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:295" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:294" ] }, { "annotation": null, "id": "r:264", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:295" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 100 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 34 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 34 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:263", "results": [ "s:294" ] }, { "annotation": null, "id": "r:265", "op": "linear.concat", "operands": [ "s:296", "s:312" ], "params": {}, "parent": null, "results": [ "s:295" ] }, { "annotation": null, "id": "r:266", "op": "attention_ops.get_prev_value", "operands": [ "s:302", "s:7" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:296" ] }, { "annotation": null, "id": "r:267", "op": "swiglu.sequence_ops.IndexedRegion.token_at", "operands": [ "s:308" ], "params": {}, "parent": null, "results": [ "s:302" ] }, { "annotation": null, "id": "r:268", "op": "attention_ops.attend_argmax_dot", "operands": [ "s:303", "s:138", "s:163" ], "params": { "assert_hardness_gt": null, "match_gain": 2000000.0, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": "r:267", "results": [ "s:302" ] }, { "annotation": null, "id": "r:269", "op": "linear.concat", "operands": [ "s:304", "s:310" ], "params": {}, "parent": "r:267", "results": [ "s:303" ] }, { "annotation": null, "id": "r:270", "op": "linear.bool_to_01", "operands": [ "s:306" ], "params": {}, "parent": "r:267", "results": [ "s:304" ] }, { "annotation": null, "id": "r:271", "op": "linear.multiply_const", "operands": [ "s:305" ], "params": { "scalar": 0.5 }, "parent": "r:270", "results": [ "s:304" ] }, { "annotation": null, "id": "r:272", "op": "linear.add_const", "operands": [ "s:306" ], "params": { "scalar": 1.0 }, "parent": "r:270", "results": [ "s:305" ] }, { "annotation": null, "id": "r:273", "op": "swiglu.map_select.in_range", "operands": [ "s:308", "s:309" ], "params": { "n_slots": 3 }, "parent": "r:267", "results": [ "s:306" ] }, { "annotation": null, "id": "r:274", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:307" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 12 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:273", "results": [ "s:306" ] }, { "annotation": null, "id": "r:275", "op": "linear.add_const", "operands": [ "s:115" ], "params": { "scalar": -1.0 }, "parent": null, "results": [ "s:308" ] }, { "annotation": null, "id": "r:276", "op": "linear.add_const", "operands": [ "s:308" ], "params": { "scalar": 1.0 }, "parent": "r:267", "results": [ "s:309" ] }, { "annotation": null, "id": "r:277", "op": "linear.add_const", "operands": [ "s:311" ], "params": { "scalar": 1.0 }, "parent": "r:267", "results": [ "s:310" ] }, { "annotation": null, "id": "r:278", "op": "attention_ops.get_prev_value", "operands": [ "s:318", "s:7" ], "params": { "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": null, "results": [ "s:312" ] }, { "annotation": null, "id": "r:279", "op": "swiglu.sequence_ops.IndexedRegion.token_at", "operands": [ "s:324" ], "params": {}, "parent": null, "results": [ "s:318" ] }, { "annotation": null, "id": "r:280", "op": "attention_ops.attend_argmax_dot", "operands": [ "s:319", "s:201", "s:219" ], "params": { "assert_hardness_gt": null, "match_gain": 2000000.0, "rope": { "base": 500000.0, "d_head": 64, "d_rot": 64, "max_positions": 512 } }, "parent": "r:279", "results": [ "s:318" ] }, { "annotation": null, "id": "r:281", "op": "linear.concat", "operands": [ "s:320", "s:326" ], "params": {}, "parent": "r:279", "results": [ "s:319" ] }, { "annotation": null, "id": "r:282", "op": "linear.bool_to_01", "operands": [ "s:322" ], "params": {}, "parent": "r:279", "results": [ "s:320" ] }, { "annotation": null, "id": "r:283", "op": "linear.multiply_const", "operands": [ "s:321" ], "params": { "scalar": 0.5 }, "parent": "r:282", "results": [ "s:320" ] }, { "annotation": null, "id": "r:284", "op": "linear.add_const", "operands": [ "s:322" ], "params": { "scalar": 1.0 }, "parent": "r:282", "results": [ "s:321" ] }, { "annotation": null, "id": "r:285", "op": "swiglu.map_select.in_range", "operands": [ "s:324", "s:325" ], "params": { "n_slots": 3 }, "parent": "r:279", "results": [ "s:322" ] }, { "annotation": null, "id": "r:286", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:323" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 12 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 12, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:285", "results": [ "s:322" ] }, { "annotation": null, "id": "r:287", "op": "linear.add_const", "operands": [ "s:179" ], "params": { "scalar": -1.0 }, "parent": null, "results": [ "s:324" ] }, { "annotation": null, "id": "r:288", "op": "linear.add_const", "operands": [ "s:324" ], "params": { "scalar": 1.0 }, "parent": "r:279", "results": [ "s:325" ] }, { "annotation": null, "id": "r:289", "op": "linear.add_const", "operands": [ "s:327" ], "params": { "scalar": 1.0 }, "parent": "r:279", "results": [ "s:326" ] }, { "annotation": null, "id": "r:290", "op": "linear.add_const", "operands": [ "s:290" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:332" ] }, { "annotation": null, "id": "r:291", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:334" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:333" ] }, { "annotation": null, "id": "r:292", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:334" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 9 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 9, 6 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 9, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:291", "results": [ "s:333" ] }, { "annotation": null, "id": "r:293", "op": "linear.concat", "operands": [ "s:335", "s:387" ], "params": {}, "parent": null, "results": [ "s:334" ] }, { "annotation": null, "id": "r:294", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:336" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:335" ] }, { "annotation": null, "id": "r:295", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:336" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 9 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 9, 6 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 9, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:294", "results": [ "s:335" ] }, { "annotation": null, "id": "r:296", "op": "linear.concat", "operands": [ "s:337", "s:360" ], "params": {}, "parent": null, "results": [ "s:336" ] }, { "annotation": null, "id": "r:297", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:338" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:337" ] }, { "annotation": null, "id": "r:298", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:338" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 100 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 34 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:297", "results": [ "s:337" ] }, { "annotation": null, "id": "r:299", "op": "linear.concat", "operands": [ "s:339", "s:359" ], "params": {}, "parent": null, "results": [ "s:338" ] }, { "annotation": null, "id": "r:300", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:340" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:339" ] }, { "annotation": null, "id": "r:301", "op": "linear.bool_to_01", "operands": [ "s:342" ], "params": {}, "parent": null, "results": [ "s:340" ] }, { "annotation": null, "id": "r:302", "op": "linear.multiply_const", "operands": [ "s:341" ], "params": { "scalar": 0.5 }, "parent": "r:301", "results": [ "s:340" ] }, { "annotation": null, "id": "r:303", "op": "linear.add_const", "operands": [ "s:342" ], "params": { "scalar": 1.0 }, "parent": "r:301", "results": [ "s:341" ] }, { "annotation": null, "id": "r:304", "op": "swiglu.map_select.in_range", "operands": [ "s:344", "s:358" ], "params": { "n_slots": 100 }, "parent": null, "results": [ "s:342" ] }, { "annotation": null, "id": "r:305", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:343" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 400 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 400, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 100 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 400, 100 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:304", "results": [ "s:342" ] }, { "annotation": null, "id": "r:306", "op": "linear.sum_nodes", "operands": [ "s:346", "s:350", "s:354", "s:356", "s:357" ], "params": { "max_fanout": null }, "parent": null, "results": [ "s:344" ] }, { "annotation": null, "id": "r:307", "op": "linear.slice_columns", "operands": [ "s:348" ], "params": { "name": "pp_ones", "start": 17, "width": 17 }, "parent": null, "results": [ "s:347" ] }, { "annotation": null, "id": "r:308", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:349" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:348" ] }, { "annotation": null, "id": "r:309", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:349" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 100 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 34 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 34 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:308", "results": [ "s:348" ] }, { "annotation": null, "id": "r:310", "op": "linear.concat", "operands": [ "s:261", "s:312" ], "params": {}, "parent": null, "results": [ "s:349" ] }, { "annotation": null, "id": "r:311", "op": "linear.slice_columns", "operands": [ "s:352" ], "params": { "name": "pp_ones", "start": 17, "width": 17 }, "parent": null, "results": [ "s:351" ] }, { "annotation": null, "id": "r:312", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:353" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:352" ] }, { "annotation": null, "id": "r:313", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:353" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 100 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 34 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 34 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:312", "results": [ "s:352" ] }, { "annotation": null, "id": "r:314", "op": "linear.concat", "operands": [ "s:296", "s:241" ], "params": {}, "parent": null, "results": [ "s:353" ] }, { "annotation": null, "id": "r:315", "op": "linear.slice_columns", "operands": [ "s:294" ], "params": { "name": "pp_tens", "start": 0, "width": 17 }, "parent": null, "results": [ "s:355" ] }, { "annotation": null, "id": "r:316", "op": "linear.add_const", "operands": [ "s:344" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:358" ] }, { "annotation": null, "id": "r:317", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:286" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:359" ] }, { "annotation": null, "id": "r:318", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:361" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:360" ] }, { "annotation": null, "id": "r:319", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:361" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 100 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 34 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:318", "results": [ "s:360" ] }, { "annotation": null, "id": "r:320", "op": "linear.concat", "operands": [ "s:362", "s:386" ], "params": {}, "parent": null, "results": [ "s:361" ] }, { "annotation": null, "id": "r:321", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:363" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:362" ] }, { "annotation": null, "id": "r:322", "op": "linear.bool_to_01", "operands": [ "s:365" ], "params": {}, "parent": null, "results": [ "s:363" ] }, { "annotation": null, "id": "r:323", "op": "linear.multiply_const", "operands": [ "s:364" ], "params": { "scalar": 0.5 }, "parent": "r:322", "results": [ "s:363" ] }, { "annotation": null, "id": "r:324", "op": "linear.add_const", "operands": [ "s:365" ], "params": { "scalar": 1.0 }, "parent": "r:322", "results": [ "s:364" ] }, { "annotation": null, "id": "r:325", "op": "swiglu.map_select.in_range", "operands": [ "s:367", "s:385" ], "params": { "n_slots": 100 }, "parent": null, "results": [ "s:365" ] }, { "annotation": null, "id": "r:326", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:366" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 400 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 400, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 100 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 400, 100 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:325", "results": [ "s:365" ] }, { "annotation": null, "id": "r:327", "op": "linear.sum_nodes", "operands": [ "s:369", "s:373", "s:377", "s:379", "s:383" ], "params": { "max_fanout": null }, "parent": null, "results": [ "s:367" ] }, { "annotation": null, "id": "r:328", "op": "linear.slice_columns", "operands": [ "s:371" ], "params": { "name": "pp_ones", "start": 17, "width": 17 }, "parent": null, "results": [ "s:370" ] }, { "annotation": null, "id": "r:329", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:372" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:371" ] }, { "annotation": null, "id": "r:330", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:372" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 100 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 34 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 34 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:329", "results": [ "s:371" ] }, { "annotation": null, "id": "r:331", "op": "linear.concat", "operands": [ "s:102", "s:312" ], "params": {}, "parent": null, "results": [ "s:372" ] }, { "annotation": null, "id": "r:332", "op": "linear.slice_columns", "operands": [ "s:375" ], "params": { "name": "pp_ones", "start": 17, "width": 17 }, "parent": null, "results": [ "s:374" ] }, { "annotation": null, "id": "r:333", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:376" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:375" ] }, { "annotation": null, "id": "r:334", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:376" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 100 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 34 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 34 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:333", "results": [ "s:375" ] }, { "annotation": null, "id": "r:335", "op": "linear.concat", "operands": [ "s:261", "s:241" ], "params": {}, "parent": null, "results": [ "s:376" ] }, { "annotation": null, "id": "r:336", "op": "linear.slice_columns", "operands": [ "s:348" ], "params": { "name": "pp_tens", "start": 0, "width": 17 }, "parent": null, "results": [ "s:378" ] }, { "annotation": null, "id": "r:337", "op": "linear.slice_columns", "operands": [ "s:381" ], "params": { "name": "pp_ones", "start": 17, "width": 17 }, "parent": null, "results": [ "s:380" ] }, { "annotation": null, "id": "r:338", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:382" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:381" ] }, { "annotation": null, "id": "r:339", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:382" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 100 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 34 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 34 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:338", "results": [ "s:381" ] }, { "annotation": null, "id": "r:340", "op": "linear.concat", "operands": [ "s:296", "s:166" ], "params": {}, "parent": null, "results": [ "s:382" ] }, { "annotation": null, "id": "r:341", "op": "linear.slice_columns", "operands": [ "s:352" ], "params": { "name": "pp_tens", "start": 0, "width": 17 }, "parent": null, "results": [ "s:384" ] }, { "annotation": null, "id": "r:342", "op": "linear.add_const", "operands": [ "s:367" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:385" ] }, { "annotation": null, "id": "r:343", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:340" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:386" ] }, { "annotation": null, "id": "r:344", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:388" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:387" ] }, { "annotation": null, "id": "r:345", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:388" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 9 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 9, 6 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 9, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:344", "results": [ "s:387" ] }, { "annotation": null, "id": "r:346", "op": "linear.concat", "operands": [ "s:389", "s:410" ], "params": {}, "parent": null, "results": [ "s:388" ] }, { "annotation": null, "id": "r:347", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:390" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:389" ] }, { "annotation": null, "id": "r:348", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:390" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 100 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 34 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:347", "results": [ "s:389" ] }, { "annotation": null, "id": "r:349", "op": "linear.concat", "operands": [ "s:391", "s:409" ], "params": {}, "parent": null, "results": [ "s:390" ] }, { "annotation": null, "id": "r:350", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:392" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:391" ] }, { "annotation": null, "id": "r:351", "op": "linear.bool_to_01", "operands": [ "s:394" ], "params": {}, "parent": null, "results": [ "s:392" ] }, { "annotation": null, "id": "r:352", "op": "linear.multiply_const", "operands": [ "s:393" ], "params": { "scalar": 0.5 }, "parent": "r:351", "results": [ "s:392" ] }, { "annotation": null, "id": "r:353", "op": "linear.add_const", "operands": [ "s:394" ], "params": { "scalar": 1.0 }, "parent": "r:351", "results": [ "s:393" ] }, { "annotation": null, "id": "r:354", "op": "swiglu.map_select.in_range", "operands": [ "s:396", "s:408" ], "params": { "n_slots": 100 }, "parent": null, "results": [ "s:394" ] }, { "annotation": null, "id": "r:355", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:395" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 400 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 400, 2 ] } }, "name": "in_range", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 100 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 400, 100 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:354", "results": [ "s:394" ] }, { "annotation": null, "id": "r:356", "op": "linear.sum_nodes", "operands": [ "s:398", "s:400", "s:402", "s:404", "s:406" ], "params": { "max_fanout": null }, "parent": null, "results": [ "s:396" ] }, { "annotation": null, "id": "r:357", "op": "linear.slice_columns", "operands": [ "s:239" ], "params": { "name": "pp_ones", "start": 17, "width": 17 }, "parent": null, "results": [ "s:399" ] }, { "annotation": null, "id": "r:358", "op": "linear.slice_columns", "operands": [ "s:371" ], "params": { "name": "pp_tens", "start": 0, "width": 17 }, "parent": null, "results": [ "s:401" ] }, { "annotation": null, "id": "r:359", "op": "linear.slice_columns", "operands": [ "s:259" ], "params": { "name": "pp_ones", "start": 17, "width": 17 }, "parent": null, "results": [ "s:403" ] }, { "annotation": null, "id": "r:360", "op": "linear.slice_columns", "operands": [ "s:375" ], "params": { "name": "pp_tens", "start": 0, "width": 17 }, "parent": null, "results": [ "s:405" ] }, { "annotation": null, "id": "r:361", "op": "linear.slice_columns", "operands": [ "s:381" ], "params": { "name": "pp_tens", "start": 0, "width": 17 }, "parent": null, "results": [ "s:407" ] }, { "annotation": null, "id": "r:362", "op": "linear.add_const", "operands": [ "s:396" ], "params": { "scalar": 1.0 }, "parent": null, "results": [ "s:408" ] }, { "annotation": null, "id": "r:363", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:363" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:409" ] }, { "annotation": null, "id": "r:364", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:411" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:410" ] }, { "annotation": null, "id": "r:365", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:411" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 100 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 34 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:364", "results": [ "s:410" ] }, { "annotation": null, "id": "r:366", "op": "linear.concat", "operands": [ "s:412", "s:413" ], "params": {}, "parent": null, "results": [ "s:411" ] }, { "annotation": null, "id": "r:367", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:229" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:412" ] }, { "annotation": null, "id": "r:368", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:392" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:413" ] }, { "annotation": null, "id": "r:369", "op": "linear.bool_to_01", "operands": [ "s:416" ], "params": {}, "parent": "r:28", "results": [ "s:414" ] }, { "annotation": null, "id": "r:370", "op": "linear.multiply_const", "operands": [ "s:415" ], "params": { "scalar": 0.5 }, "parent": "r:369", "results": [ "s:414" ] }, { "annotation": null, "id": "r:371", "op": "linear.add_const", "operands": [ "s:416" ], "params": { "scalar": 1.0 }, "parent": "r:369", "results": [ "s:415" ] }, { "annotation": null, "id": "r:372", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:417" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.5, "true_level": 1.0 }, "parent": "r:28", "results": [ "s:416" ] }, { "annotation": null, "id": "r:373", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:417" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:372", "results": [ "s:416" ] }, { "annotation": null, "id": "r:374", "op": "linear.sum_nodes", "operands": [ "s:39", "s:419" ], "params": { "max_fanout": null }, "parent": "r:28", "results": [ "s:417" ] }, { "annotation": null, "id": "r:375", "op": "linear.bool_to_01", "operands": [ "s:421" ], "params": {}, "parent": "r:28", "results": [ "s:419" ] }, { "annotation": null, "id": "r:376", "op": "linear.multiply_const", "operands": [ "s:420" ], "params": { "scalar": 0.5 }, "parent": "r:375", "results": [ "s:419" ] }, { "annotation": null, "id": "r:377", "op": "linear.add_const", "operands": [ "s:421" ], "params": { "scalar": 1.0 }, "parent": "r:375", "results": [ "s:420" ] }, { "annotation": null, "id": "r:378", "op": "swiglu.logic_ops.equals_vector", "operands": [ "s:422" ], "params": { "vector": { "tensor": { "dtype": "float32", "shape": [ 17 ] } } }, "parent": "r:28", "results": [ "s:421" ] }, { "annotation": null, "id": "r:379", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:422" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 17 ] } }, "name": "equals_vector", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:378", "results": [ "s:421" ] }, { "annotation": null, "id": "r:380", "op": "swiglu.map_select.switch", "operands": [ "s:46", "s:74", "s:83", "s:71", "s:80", "s:430" ], "params": {}, "parent": null, "results": [ "s:422" ] }, { "annotation": null, "id": "r:381", "op": "linear.sum_nodes", "operands": [ "s:424", "s:426", "s:428" ], "params": { "max_fanout": null }, "parent": "r:380", "results": [ "s:422" ] }, { "annotation": null, "id": "r:382", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:46", "s:71" ], "params": {}, "parent": "r:380", "results": [ "s:424" ] }, { "annotation": null, "id": "r:383", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:425" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:382", "results": [ "s:424" ] }, { "annotation": null, "id": "r:384", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:74", "s:80" ], "params": {}, "parent": "r:380", "results": [ "s:426" ] }, { "annotation": null, "id": "r:385", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:427" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:384", "results": [ "s:426" ] }, { "annotation": null, "id": "r:386", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:83", "s:430" ], "params": {}, "parent": "r:380", "results": [ "s:428" ] }, { "annotation": null, "id": "r:387", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:429" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:386", "results": [ "s:428" ] }, { "annotation": null, "id": "r:388", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:431" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:430" ] }, { "annotation": null, "id": "r:389", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:431" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 200 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 36 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 17 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:388", "results": [ "s:430" ] }, { "annotation": null, "id": "r:390", "op": "linear.concat", "operands": [ "s:412", "s:413", "s:432" ], "params": {}, "parent": null, "results": [ "s:431" ] }, { "annotation": null, "id": "r:391", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:433" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:432" ] }, { "annotation": null, "id": "r:392", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:434" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:433" ] }, { "annotation": null, "id": "r:393", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:434" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 9 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 9, 6 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 9, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:392", "results": [ "s:433" ] }, { "annotation": null, "id": "r:394", "op": "linear.concat", "operands": [ "s:435", "s:437" ], "params": {}, "parent": null, "results": [ "s:434" ] }, { "annotation": null, "id": "r:395", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:436" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:435" ] }, { "annotation": null, "id": "r:396", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:436" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 9 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 9, 6 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 9, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:395", "results": [ "s:435" ] }, { "annotation": null, "id": "r:397", "op": "linear.concat", "operands": [ "s:283", "s:337" ], "params": {}, "parent": null, "results": [ "s:436" ] }, { "annotation": null, "id": "r:398", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:438" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:437" ] }, { "annotation": null, "id": "r:399", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:438" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 9 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 9, 6 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 9, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:398", "results": [ "s:437" ] }, { "annotation": null, "id": "r:400", "op": "linear.concat", "operands": [ "s:360", "s:389" ], "params": {}, "parent": null, "results": [ "s:438" ] }, { "annotation": null, "id": "r:401", "op": "linear.bool_to_01", "operands": [ "s:441" ], "params": {}, "parent": "r:28", "results": [ "s:439" ] }, { "annotation": null, "id": "r:402", "op": "linear.multiply_const", "operands": [ "s:440" ], "params": { "scalar": 0.5 }, "parent": "r:401", "results": [ "s:439" ] }, { "annotation": null, "id": "r:403", "op": "linear.add_const", "operands": [ "s:441" ], "params": { "scalar": 1.0 }, "parent": "r:401", "results": [ "s:440" ] }, { "annotation": null, "id": "r:404", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:442" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 2.5, "true_level": 1.0 }, "parent": "r:28", "results": [ "s:441" ] }, { "annotation": null, "id": "r:405", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:442" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:404", "results": [ "s:441" ] }, { "annotation": null, "id": "r:406", "op": "linear.sum_nodes", "operands": [ "s:39", "s:419", "s:444" ], "params": { "max_fanout": null }, "parent": "r:28", "results": [ "s:442" ] }, { "annotation": null, "id": "r:407", "op": "linear.bool_to_01", "operands": [ "s:446" ], "params": {}, "parent": "r:28", "results": [ "s:444" ] }, { "annotation": null, "id": "r:408", "op": "linear.multiply_const", "operands": [ "s:445" ], "params": { "scalar": 0.5 }, "parent": "r:407", "results": [ "s:444" ] }, { "annotation": null, "id": "r:409", "op": "linear.add_const", "operands": [ "s:446" ], "params": { "scalar": 1.0 }, "parent": "r:407", "results": [ "s:445" ] }, { "annotation": null, "id": "r:410", "op": "swiglu.logic_ops.equals_vector", "operands": [ "s:447" ], "params": { "vector": { "tensor": { "dtype": "float32", "shape": [ 17 ] } } }, "parent": "r:28", "results": [ "s:446" ] }, { "annotation": null, "id": "r:411", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:447" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 17 ] } }, "name": "equals_vector", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:410", "results": [ "s:446" ] }, { "annotation": null, "id": "r:412", "op": "swiglu.map_select.switch", "operands": [ "s:46", "s:74", "s:83", "s:451", "s:80", "s:469" ], "params": {}, "parent": null, "results": [ "s:447" ] }, { "annotation": null, "id": "r:413", "op": "linear.sum_nodes", "operands": [ "s:449", "s:465", "s:467" ], "params": { "max_fanout": null }, "parent": "r:412", "results": [ "s:447" ] }, { "annotation": null, "id": "r:414", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:46", "s:451" ], "params": {}, "parent": "r:412", "results": [ "s:449" ] }, { "annotation": null, "id": "r:415", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:450" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:414", "results": [ "s:449" ] }, { "annotation": null, "id": "r:416", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:452" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:451" ] }, { "annotation": null, "id": "r:417", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:452" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 200 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 36 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 17 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:416", "results": [ "s:451" ] }, { "annotation": null, "id": "r:418", "op": "linear.concat", "operands": [ "s:453", "s:453", "s:454" ], "params": {}, "parent": null, "results": [ "s:452" ] }, { "annotation": null, "id": "r:419", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 17 ] } } }, "parent": null, "results": [ "s:453" ] }, { "annotation": null, "id": "r:420", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:455" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:454" ] }, { "annotation": null, "id": "r:421", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:456" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:455" ] }, { "annotation": null, "id": "r:422", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:456" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 9 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 9, 6 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 9, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:421", "results": [ "s:455" ] }, { "annotation": null, "id": "r:423", "op": "linear.concat", "operands": [ "s:457", "s:459" ], "params": {}, "parent": null, "results": [ "s:456" ] }, { "annotation": null, "id": "r:424", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:458" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:457" ] }, { "annotation": null, "id": "r:425", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:458" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 100 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 34 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:424", "results": [ "s:457" ] }, { "annotation": null, "id": "r:426", "op": "linear.concat", "operands": [ "s:296", "s:312" ], "params": {}, "parent": null, "results": [ "s:458" ] }, { "annotation": null, "id": "r:427", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:460" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:459" ] }, { "annotation": null, "id": "r:428", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:460" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 9 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 9, 6 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 9, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:427", "results": [ "s:459" ] }, { "annotation": null, "id": "r:429", "op": "linear.concat", "operands": [ "s:461", "s:463" ], "params": {}, "parent": null, "results": [ "s:460" ] }, { "annotation": null, "id": "r:430", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:462" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:461" ] }, { "annotation": null, "id": "r:431", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:462" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 100 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 34 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:430", "results": [ "s:461" ] }, { "annotation": null, "id": "r:432", "op": "linear.concat", "operands": [ "s:261", "s:241" ], "params": {}, "parent": null, "results": [ "s:462" ] }, { "annotation": null, "id": "r:433", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:464" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:463" ] }, { "annotation": null, "id": "r:434", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:464" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 100 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 34 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:433", "results": [ "s:463" ] }, { "annotation": null, "id": "r:435", "op": "linear.concat", "operands": [ "s:102", "s:166" ], "params": {}, "parent": null, "results": [ "s:464" ] }, { "annotation": null, "id": "r:436", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:74", "s:80" ], "params": {}, "parent": "r:412", "results": [ "s:465" ] }, { "annotation": null, "id": "r:437", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:466" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:436", "results": [ "s:465" ] }, { "annotation": null, "id": "r:438", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:83", "s:469" ], "params": {}, "parent": "r:412", "results": [ "s:467" ] }, { "annotation": null, "id": "r:439", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:468" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:438", "results": [ "s:467" ] }, { "annotation": null, "id": "r:440", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:470" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:469" ] }, { "annotation": null, "id": "r:441", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:470" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 200 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 36 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 17 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:440", "results": [ "s:469" ] }, { "annotation": null, "id": "r:442", "op": "linear.concat", "operands": [ "s:391", "s:409", "s:471" ], "params": {}, "parent": null, "results": [ "s:470" ] }, { "annotation": null, "id": "r:443", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:472" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:471" ] }, { "annotation": null, "id": "r:444", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:473" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:472" ] }, { "annotation": null, "id": "r:445", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:473" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 9 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 9, 6 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 9, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:444", "results": [ "s:472" ] }, { "annotation": null, "id": "r:446", "op": "linear.concat", "operands": [ "s:283", "s:335" ], "params": {}, "parent": null, "results": [ "s:473" ] }, { "annotation": null, "id": "r:447", "op": "linear.bool_to_01", "operands": [ "s:476" ], "params": {}, "parent": "r:28", "results": [ "s:474" ] }, { "annotation": null, "id": "r:448", "op": "linear.multiply_const", "operands": [ "s:475" ], "params": { "scalar": 0.5 }, "parent": "r:447", "results": [ "s:474" ] }, { "annotation": null, "id": "r:449", "op": "linear.add_const", "operands": [ "s:476" ], "params": { "scalar": 1.0 }, "parent": "r:447", "results": [ "s:475" ] }, { "annotation": null, "id": "r:450", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:477" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 3.5, "true_level": 1.0 }, "parent": "r:28", "results": [ "s:476" ] }, { "annotation": null, "id": "r:451", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:477" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:450", "results": [ "s:476" ] }, { "annotation": null, "id": "r:452", "op": "linear.sum_nodes", "operands": [ "s:39", "s:419", "s:444", "s:479" ], "params": { "max_fanout": null }, "parent": "r:28", "results": [ "s:477" ] }, { "annotation": null, "id": "r:453", "op": "linear.bool_to_01", "operands": [ "s:481" ], "params": {}, "parent": "r:28", "results": [ "s:479" ] }, { "annotation": null, "id": "r:454", "op": "linear.multiply_const", "operands": [ "s:480" ], "params": { "scalar": 0.5 }, "parent": "r:453", "results": [ "s:479" ] }, { "annotation": null, "id": "r:455", "op": "linear.add_const", "operands": [ "s:481" ], "params": { "scalar": 1.0 }, "parent": "r:453", "results": [ "s:480" ] }, { "annotation": null, "id": "r:456", "op": "swiglu.logic_ops.equals_vector", "operands": [ "s:482" ], "params": { "vector": { "tensor": { "dtype": "float32", "shape": [ 17 ] } } }, "parent": "r:28", "results": [ "s:481" ] }, { "annotation": null, "id": "r:457", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:482" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 17 ] } }, "name": "equals_vector", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:456", "results": [ "s:481" ] }, { "annotation": null, "id": "r:458", "op": "swiglu.map_select.switch", "operands": [ "s:46", "s:74", "s:83", "s:486", "s:493", "s:531" ], "params": {}, "parent": null, "results": [ "s:482" ] }, { "annotation": null, "id": "r:459", "op": "linear.sum_nodes", "operands": [ "s:484", "s:491", "s:529" ], "params": { "max_fanout": null }, "parent": "r:458", "results": [ "s:482" ] }, { "annotation": null, "id": "r:460", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:46", "s:486" ], "params": {}, "parent": "r:458", "results": [ "s:484" ] }, { "annotation": null, "id": "r:461", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:485" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:460", "results": [ "s:484" ] }, { "annotation": null, "id": "r:462", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:487" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:486" ] }, { "annotation": null, "id": "r:463", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:487" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 200 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 36 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 17 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:462", "results": [ "s:486" ] }, { "annotation": null, "id": "r:464", "op": "linear.concat", "operands": [ "s:102", "s:166", "s:488" ], "params": {}, "parent": null, "results": [ "s:487" ] }, { "annotation": null, "id": "r:465", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:489" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:488" ] }, { "annotation": null, "id": "r:466", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:490" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:489" ] }, { "annotation": null, "id": "r:467", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:490" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 9 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 9, 6 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 9, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:466", "results": [ "s:489" ] }, { "annotation": null, "id": "r:468", "op": "linear.concat", "operands": [ "s:457", "s:461" ], "params": {}, "parent": null, "results": [ "s:490" ] }, { "annotation": null, "id": "r:469", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:74", "s:493" ], "params": {}, "parent": "r:458", "results": [ "s:491" ] }, { "annotation": null, "id": "r:470", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:492" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:469", "results": [ "s:491" ] }, { "annotation": null, "id": "r:471", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:494" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:493" ] }, { "annotation": null, "id": "r:472", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:494" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 200 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 36 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 17 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:471", "results": [ "s:493" ] }, { "annotation": null, "id": "r:473", "op": "linear.concat", "operands": [ "s:495", "s:512", "s:514" ], "params": {}, "parent": null, "results": [ "s:494" ] }, { "annotation": null, "id": "r:474", "op": "swiglu.map_select.select", "operands": [ "s:497", "s:102", "s:166" ], "params": {}, "parent": null, "results": [ "s:495" ] }, { "annotation": null, "id": "r:475", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:496" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 35 ] } }, "name": "select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 35 ] } } }, "parent": "r:474", "results": [ "s:495" ] }, { "annotation": null, "id": "r:476", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:498" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -0.5, "true_level": 1.0 }, "parent": null, "results": [ "s:497" ] }, { "annotation": null, "id": "r:477", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:498" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:476", "results": [ "s:497" ] }, { "annotation": null, "id": "r:478", "op": "linear.concat", "operands": [ "s:500", "s:503", "s:504", "s:507", "s:508", "s:511" ], "params": {}, "parent": null, "results": [ "s:499" ] }, { "annotation": null, "id": "r:479", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:501" ], "params": { "false_level": 0.0, "sharpness": null, "thresh": 0.5, "true_level": 1.0 }, "parent": null, "results": [ "s:500" ] }, { "annotation": null, "id": "r:480", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:501" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:479", "results": [ "s:500" ] }, { "annotation": null, "id": "r:481", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:502" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:501" ] }, { "annotation": null, "id": "r:482", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:502" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 100 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 34 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:481", "results": [ "s:501" ] }, { "annotation": null, "id": "r:483", "op": "linear.concat", "operands": [ "s:102", "s:166" ], "params": {}, "parent": null, "results": [ "s:502" ] }, { "annotation": null, "id": "r:484", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:501" ], "params": { "false_level": 1.0, "sharpness": null, "thresh": -0.5, "true_level": 0.0 }, "parent": null, "results": [ "s:503" ] }, { "annotation": null, "id": "r:485", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:501" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:484", "results": [ "s:503" ] }, { "annotation": null, "id": "r:486", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:505" ], "params": { "false_level": 0.0, "sharpness": null, "thresh": 0.5, "true_level": 1.0 }, "parent": null, "results": [ "s:504" ] }, { "annotation": null, "id": "r:487", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:505" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:486", "results": [ "s:504" ] }, { "annotation": null, "id": "r:488", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:506" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:505" ] }, { "annotation": null, "id": "r:489", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:506" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 100 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 34 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:488", "results": [ "s:505" ] }, { "annotation": null, "id": "r:490", "op": "linear.concat", "operands": [ "s:261", "s:241" ], "params": {}, "parent": null, "results": [ "s:506" ] }, { "annotation": null, "id": "r:491", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:505" ], "params": { "false_level": 1.0, "sharpness": null, "thresh": -0.5, "true_level": 0.0 }, "parent": null, "results": [ "s:507" ] }, { "annotation": null, "id": "r:492", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:505" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:491", "results": [ "s:507" ] }, { "annotation": null, "id": "r:493", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:509" ], "params": { "false_level": 0.0, "sharpness": null, "thresh": 0.5, "true_level": 1.0 }, "parent": null, "results": [ "s:508" ] }, { "annotation": null, "id": "r:494", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:509" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:493", "results": [ "s:508" ] }, { "annotation": null, "id": "r:495", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:510" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:509" ] }, { "annotation": null, "id": "r:496", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:510" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 100 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 34 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:495", "results": [ "s:509" ] }, { "annotation": null, "id": "r:497", "op": "linear.concat", "operands": [ "s:296", "s:312" ], "params": {}, "parent": null, "results": [ "s:510" ] }, { "annotation": null, "id": "r:498", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:509" ], "params": { "false_level": 1.0, "sharpness": null, "thresh": -0.5, "true_level": 0.0 }, "parent": null, "results": [ "s:511" ] }, { "annotation": null, "id": "r:499", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:509" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:498", "results": [ "s:511" ] }, { "annotation": null, "id": "r:500", "op": "swiglu.map_select.select", "operands": [ "s:497", "s:166", "s:102" ], "params": {}, "parent": null, "results": [ "s:512" ] }, { "annotation": null, "id": "r:501", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:513" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 35 ] } }, "name": "select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 35 ] } } }, "parent": "r:500", "results": [ "s:512" ] }, { "annotation": null, "id": "r:502", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:515" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:514" ] }, { "annotation": null, "id": "r:503", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:516" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:515" ] }, { "annotation": null, "id": "r:504", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:516" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 9 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 9, 6 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 9, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:503", "results": [ "s:515" ] }, { "annotation": null, "id": "r:505", "op": "linear.concat", "operands": [ "s:517", "s:523" ], "params": {}, "parent": null, "results": [ "s:516" ] }, { "annotation": null, "id": "r:506", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:518" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:517" ] }, { "annotation": null, "id": "r:507", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:518" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 100 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 34 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:506", "results": [ "s:517" ] }, { "annotation": null, "id": "r:508", "op": "linear.concat", "operands": [ "s:519", "s:521" ], "params": {}, "parent": null, "results": [ "s:518" ] }, { "annotation": null, "id": "r:509", "op": "swiglu.map_select.select", "operands": [ "s:497", "s:296", "s:312" ], "params": {}, "parent": null, "results": [ "s:519" ] }, { "annotation": null, "id": "r:510", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:520" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 35 ] } }, "name": "select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 35 ] } } }, "parent": "r:509", "results": [ "s:519" ] }, { "annotation": null, "id": "r:511", "op": "swiglu.map_select.select", "operands": [ "s:497", "s:312", "s:296" ], "params": {}, "parent": null, "results": [ "s:521" ] }, { "annotation": null, "id": "r:512", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:522" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 35 ] } }, "name": "select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 35 ] } } }, "parent": "r:511", "results": [ "s:521" ] }, { "annotation": null, "id": "r:513", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:524" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:523" ] }, { "annotation": null, "id": "r:514", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:524" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 100 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 34 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 3 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 100, 3 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:513", "results": [ "s:523" ] }, { "annotation": null, "id": "r:515", "op": "linear.concat", "operands": [ "s:525", "s:527" ], "params": {}, "parent": null, "results": [ "s:524" ] }, { "annotation": null, "id": "r:516", "op": "swiglu.map_select.select", "operands": [ "s:497", "s:261", "s:241" ], "params": {}, "parent": null, "results": [ "s:525" ] }, { "annotation": null, "id": "r:517", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:526" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 35 ] } }, "name": "select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 35 ] } } }, "parent": "r:516", "results": [ "s:525" ] }, { "annotation": null, "id": "r:518", "op": "swiglu.map_select.select", "operands": [ "s:497", "s:241", "s:261" ], "params": {}, "parent": null, "results": [ "s:527" ] }, { "annotation": null, "id": "r:519", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:528" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 35 ] } }, "name": "select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 34 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 34, 35 ] } } }, "parent": "r:518", "results": [ "s:527" ] }, { "annotation": null, "id": "r:520", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:83", "s:531" ], "params": {}, "parent": "r:458", "results": [ "s:529" ] }, { "annotation": null, "id": "r:521", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:530" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:520", "results": [ "s:529" ] }, { "annotation": null, "id": "r:522", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:532" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:531" ] }, { "annotation": null, "id": "r:523", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:532" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 200 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 36 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 17 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:522", "results": [ "s:531" ] }, { "annotation": null, "id": "r:524", "op": "linear.concat", "operands": [ "s:362", "s:386", "s:533" ], "params": {}, "parent": null, "results": [ "s:532" ] }, { "annotation": null, "id": "r:525", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:435" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:533" ] }, { "annotation": null, "id": "r:526", "op": "linear.bool_to_01", "operands": [ "s:536" ], "params": {}, "parent": "r:28", "results": [ "s:534" ] }, { "annotation": null, "id": "r:527", "op": "linear.multiply_const", "operands": [ "s:535" ], "params": { "scalar": 0.5 }, "parent": "r:526", "results": [ "s:534" ] }, { "annotation": null, "id": "r:528", "op": "linear.add_const", "operands": [ "s:536" ], "params": { "scalar": 1.0 }, "parent": "r:526", "results": [ "s:535" ] }, { "annotation": null, "id": "r:529", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:537" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 4.5, "true_level": 1.0 }, "parent": "r:28", "results": [ "s:536" ] }, { "annotation": null, "id": "r:530", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:537" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:529", "results": [ "s:536" ] }, { "annotation": null, "id": "r:531", "op": "linear.sum_nodes", "operands": [ "s:39", "s:419", "s:444", "s:479", "s:539" ], "params": { "max_fanout": null }, "parent": "r:28", "results": [ "s:537" ] }, { "annotation": null, "id": "r:532", "op": "linear.bool_to_01", "operands": [ "s:541" ], "params": {}, "parent": "r:28", "results": [ "s:539" ] }, { "annotation": null, "id": "r:533", "op": "linear.multiply_const", "operands": [ "s:540" ], "params": { "scalar": 0.5 }, "parent": "r:532", "results": [ "s:539" ] }, { "annotation": null, "id": "r:534", "op": "linear.add_const", "operands": [ "s:541" ], "params": { "scalar": 1.0 }, "parent": "r:532", "results": [ "s:540" ] }, { "annotation": null, "id": "r:535", "op": "swiglu.logic_ops.equals_vector", "operands": [ "s:542" ], "params": { "vector": { "tensor": { "dtype": "float32", "shape": [ 17 ] } } }, "parent": "r:28", "results": [ "s:541" ] }, { "annotation": null, "id": "r:536", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:542" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 17 ] } }, "name": "equals_vector", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 1, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:535", "results": [ "s:541" ] }, { "annotation": null, "id": "r:537", "op": "swiglu.map_select.switch", "operands": [ "s:46", "s:74", "s:83", "s:546", "s:551", "s:556" ], "params": {}, "parent": null, "results": [ "s:542" ] }, { "annotation": null, "id": "r:538", "op": "linear.sum_nodes", "operands": [ "s:544", "s:549", "s:554" ], "params": { "max_fanout": null }, "parent": "r:537", "results": [ "s:542" ] }, { "annotation": null, "id": "r:539", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:46", "s:546" ], "params": {}, "parent": "r:537", "results": [ "s:544" ] }, { "annotation": null, "id": "r:540", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:545" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:539", "results": [ "s:544" ] }, { "annotation": null, "id": "r:541", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:547" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:546" ] }, { "annotation": null, "id": "r:542", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:547" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 200 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 36 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 17 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:541", "results": [ "s:546" ] }, { "annotation": null, "id": "r:543", "op": "linear.concat", "operands": [ "s:261", "s:241", "s:548" ], "params": {}, "parent": null, "results": [ "s:547" ] }, { "annotation": null, "id": "r:544", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:457" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:548" ] }, { "annotation": null, "id": "r:545", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:74", "s:551" ], "params": {}, "parent": "r:537", "results": [ "s:549" ] }, { "annotation": null, "id": "r:546", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:550" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:545", "results": [ "s:549" ] }, { "annotation": null, "id": "r:547", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:552" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:551" ] }, { "annotation": null, "id": "r:548", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:552" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 200 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 36 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 17 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:547", "results": [ "s:551" ] }, { "annotation": null, "id": "r:549", "op": "linear.concat", "operands": [ "s:525", "s:527", "s:553" ], "params": {}, "parent": null, "results": [ "s:552" ] }, { "annotation": null, "id": "r:550", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:517" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:553" ] }, { "annotation": null, "id": "r:551", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:83", "s:556" ], "params": {}, "parent": "r:537", "results": [ "s:554" ] }, { "annotation": null, "id": "r:552", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:555" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:551", "results": [ "s:554" ] }, { "annotation": null, "id": "r:553", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:557" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:556" ] }, { "annotation": null, "id": "r:554", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:557" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 200 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 36 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 17 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:553", "results": [ "s:556" ] }, { "annotation": null, "id": "r:555", "op": "linear.concat", "operands": [ "s:339", "s:359", "s:558" ], "params": {}, "parent": null, "results": [ "s:557" ] }, { "annotation": null, "id": "r:556", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:283" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:558" ] }, { "annotation": null, "id": "r:557", "op": "linear.multiply_const", "operands": [ "s:560" ], "params": { "scalar": 6.0 }, "parent": "r:28", "results": [ "s:559" ] }, { "annotation": null, "id": "r:558", "op": "linear.bool_to_01", "operands": [ "s:562" ], "params": {}, "parent": "r:28", "results": [ "s:560" ] }, { "annotation": null, "id": "r:559", "op": "linear.multiply_const", "operands": [ "s:561" ], "params": { "scalar": 0.5 }, "parent": "r:558", "results": [ "s:560" ] }, { "annotation": null, "id": "r:560", "op": "linear.add_const", "operands": [ "s:562" ], "params": { "scalar": 1.0 }, "parent": "r:558", "results": [ "s:561" ] }, { "annotation": null, "id": "r:561", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:74", "s:565" ], "params": {}, "parent": null, "results": [ "s:562" ] }, { "annotation": null, "id": "r:562", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:563" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:561", "results": [ "s:562" ] }, { "annotation": null, "id": "r:563", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:563" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:562", "results": [ "s:562" ] }, { "annotation": null, "id": "r:564", "op": "linear.sum_nodes", "operands": [ "s:74", "s:565" ], "params": { "max_fanout": null }, "parent": "r:561", "results": [ "s:563" ] }, { "annotation": null, "id": "r:565", "op": "swiglu.logic_ops.bool_not", "operands": [ "s:497" ], "params": {}, "parent": null, "results": [ "s:565" ] }, { "annotation": null, "id": "r:566", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:497" ], "params": { "false_level": 1.0, "sharpness": null, "thresh": 0.0, "true_level": -1.0 }, "parent": "r:565", "results": [ "s:565" ] }, { "annotation": null, "id": "r:567", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:497" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:566", "results": [ "s:565" ] }, { "annotation": null, "id": "r:568", "op": "linear.add_const", "operands": [ "s:31" ], "params": { "scalar": 1.0 }, "parent": "r:28", "results": [ "s:566" ] }, { "annotation": null, "id": "r:569", "op": "linear.concat", "operands": [ "s:42", "s:422", "s:447", "s:482", "s:542", "s:568", "s:585", "s:585", "s:585", "s:585", "s:585", "s:585" ], "params": {}, "parent": "r:28", "results": [ "s:567" ] }, { "annotation": null, "id": "r:570", "op": "swiglu.map_select.switch", "operands": [ "s:46", "s:74", "s:83", "s:572", "s:577", "s:582" ], "params": {}, "parent": null, "results": [ "s:568" ] }, { "annotation": null, "id": "r:571", "op": "linear.sum_nodes", "operands": [ "s:570", "s:575", "s:580" ], "params": { "max_fanout": null }, "parent": "r:570", "results": [ "s:568" ] }, { "annotation": null, "id": "r:572", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:46", "s:572" ], "params": {}, "parent": "r:570", "results": [ "s:570" ] }, { "annotation": null, "id": "r:573", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:571" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:572", "results": [ "s:570" ] }, { "annotation": null, "id": "r:574", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:573" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:572" ] }, { "annotation": null, "id": "r:575", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:573" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 200 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 36 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 17 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:574", "results": [ "s:572" ] }, { "annotation": null, "id": "r:576", "op": "linear.concat", "operands": [ "s:296", "s:312", "s:574" ], "params": {}, "parent": null, "results": [ "s:573" ] }, { "annotation": null, "id": "r:577", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 2 ] } } }, "parent": null, "results": [ "s:574" ] }, { "annotation": null, "id": "r:578", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:74", "s:577" ], "params": {}, "parent": "r:570", "results": [ "s:575" ] }, { "annotation": null, "id": "r:579", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:576" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:578", "results": [ "s:575" ] }, { "annotation": null, "id": "r:580", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:578" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:577" ] }, { "annotation": null, "id": "r:581", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:578" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 200 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 36 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 17 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:580", "results": [ "s:577" ] }, { "annotation": null, "id": "r:582", "op": "linear.concat", "operands": [ "s:519", "s:521", "s:579" ], "params": {}, "parent": null, "results": [ "s:578" ] }, { "annotation": null, "id": "r:583", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 2 ] } } }, "parent": null, "results": [ "s:579" ] }, { "annotation": null, "id": "r:584", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:83", "s:582" ], "params": {}, "parent": "r:570", "results": [ "s:580" ] }, { "annotation": null, "id": "r:585", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:581" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:584", "results": [ "s:580" ] }, { "annotation": null, "id": "r:586", "op": "swiglu.onehot_table.onehot_lookup", "operands": [ "s:583" ], "params": { "default": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "key_to_value": "" }, "parent": null, "results": [ "s:582" ] }, { "annotation": null, "id": "r:587", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:583" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 200 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 36 ] } }, "name": "onehot_lookup", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 200, 17 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:586", "results": [ "s:582" ] }, { "annotation": null, "id": "r:588", "op": "linear.concat", "operands": [ "s:285", "s:223", "s:584" ], "params": {}, "parent": null, "results": [ "s:583" ] }, { "annotation": null, "id": "r:589", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 2 ] } } }, "parent": null, "results": [ "s:584" ] }, { "annotation": null, "id": "r:590", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "remove_leading_0s_sign", "vector": { "tensor": { "dtype": "float32", "shape": [ 17 ] } } }, "parent": "r:28", "results": [ "s:585" ] }, { "annotation": null, "id": "r:591", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:588", "s:594" ], "params": {}, "parent": "r:0", "results": [ "s:586" ] }, { "annotation": null, "id": "r:592", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:587" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:591", "results": [ "s:586" ] }, { "annotation": null, "id": "r:593", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:591", "s:592" ], "params": {}, "parent": "r:0", "results": [ "s:588" ] }, { "annotation": null, "id": "r:594", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:589" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:593", "results": [ "s:588" ] }, { "annotation": null, "id": "r:595", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:589" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:594", "results": [ "s:588" ] }, { "annotation": null, "id": "r:596", "op": "linear.sum_nodes", "operands": [ "s:591", "s:592" ], "params": { "max_fanout": null }, "parent": "r:593", "results": [ "s:589" ] }, { "annotation": null, "id": "r:597", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:18" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 0.5, "true_level": 1.0 }, "parent": "r:0", "results": [ "s:591" ] }, { "annotation": null, "id": "r:598", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:18" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:597", "results": [ "s:591" ] }, { "annotation": null, "id": "r:599", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:593" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -1.5, "true_level": 1.0 }, "parent": "r:0", "results": [ "s:592" ] }, { "annotation": null, "id": "r:600", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:593" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:599", "results": [ "s:592" ] }, { "annotation": null, "id": "r:601", "op": "linear.negate", "operands": [ "s:18" ], "params": {}, "parent": "r:0", "results": [ "s:593" ] }, { "annotation": null, "id": "r:602", "op": "swiglu.map_select.broadcast_select", "operands": [ "s:29", "s:597" ], "params": { "d_fill": 17, "n_slots": 12 }, "parent": "r:28", "results": [ "s:595" ] }, { "annotation": null, "id": "r:603", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:596" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 204 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 204, 216 ] } }, "name": "broadcast_select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 204 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 204, 204 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 204 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 204, 216 ] } } }, "parent": "r:602", "results": [ "s:595" ] }, { "annotation": null, "id": "r:604", "op": "linear.concat", "operands": [ "s:422", "s:447", "s:482", "s:542", "s:568", "s:598", "s:42", "s:422", "s:447", "s:482", "s:542", "s:568" ], "params": {}, "parent": "r:28", "results": [ "s:597" ] }, { "annotation": null, "id": "r:605", "op": "swiglu.map_select.switch", "operands": [ "s:46", "s:74", "s:83", "s:602", "s:605", "s:608" ], "params": {}, "parent": null, "results": [ "s:598" ] }, { "annotation": null, "id": "r:606", "op": "linear.sum_nodes", "operands": [ "s:600", "s:603", "s:606" ], "params": { "max_fanout": null }, "parent": "r:605", "results": [ "s:598" ] }, { "annotation": null, "id": "r:607", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:46", "s:602" ], "params": {}, "parent": "r:605", "results": [ "s:600" ] }, { "annotation": null, "id": "r:608", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:601" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:607", "results": [ "s:600" ] }, { "annotation": null, "id": "r:609", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 17 ] } } }, "parent": null, "results": [ "s:602" ] }, { "annotation": null, "id": "r:610", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:74", "s:605" ], "params": {}, "parent": "r:605", "results": [ "s:603" ] }, { "annotation": null, "id": "r:611", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:604" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:610", "results": [ "s:603" ] }, { "annotation": null, "id": "r:612", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 17 ] } } }, "parent": null, "results": [ "s:605" ] }, { "annotation": null, "id": "r:613", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:83", "s:608" ], "params": {}, "parent": "r:605", "results": [ "s:606" ] }, { "annotation": null, "id": "r:614", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:607" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:613", "results": [ "s:606" ] }, { "annotation": null, "id": "r:615", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 17 ] } } }, "parent": null, "results": [ "s:608" ] }, { "annotation": null, "id": "r:616", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:611", "s:617" ], "params": {}, "parent": "r:0", "results": [ "s:609" ] }, { "annotation": null, "id": "r:617", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:610" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:616", "results": [ "s:609" ] }, { "annotation": null, "id": "r:618", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:614", "s:615" ], "params": {}, "parent": "r:0", "results": [ "s:611" ] }, { "annotation": null, "id": "r:619", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:612" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:618", "results": [ "s:611" ] }, { "annotation": null, "id": "r:620", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:612" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:619", "results": [ "s:611" ] }, { "annotation": null, "id": "r:621", "op": "linear.sum_nodes", "operands": [ "s:614", "s:615" ], "params": { "max_fanout": null }, "parent": "r:618", "results": [ "s:612" ] }, { "annotation": null, "id": "r:622", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:18" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.5, "true_level": 1.0 }, "parent": "r:0", "results": [ "s:614" ] }, { "annotation": null, "id": "r:623", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:18" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:622", "results": [ "s:614" ] }, { "annotation": null, "id": "r:624", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:616" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -2.5, "true_level": 1.0 }, "parent": "r:0", "results": [ "s:615" ] }, { "annotation": null, "id": "r:625", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:616" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:624", "results": [ "s:615" ] }, { "annotation": null, "id": "r:626", "op": "linear.negate", "operands": [ "s:18" ], "params": {}, "parent": "r:0", "results": [ "s:616" ] }, { "annotation": null, "id": "r:627", "op": "swiglu.map_select.broadcast_select", "operands": [ "s:29", "s:620" ], "params": { "d_fill": 17, "n_slots": 12 }, "parent": "r:28", "results": [ "s:618" ] }, { "annotation": null, "id": "r:628", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:619" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 204 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 204, 216 ] } }, "name": "broadcast_select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 204 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 204, 204 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 204 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 204, 216 ] } } }, "parent": "r:627", "results": [ "s:618" ] }, { "annotation": null, "id": "r:629", "op": "linear.concat", "operands": [ "s:447", "s:482", "s:542", "s:568", "s:598", "s:621", "s:422", "s:447", "s:482", "s:542", "s:568", "s:598" ], "params": {}, "parent": "r:28", "results": [ "s:620" ] }, { "annotation": null, "id": "r:630", "op": "swiglu.map_select.switch", "operands": [ "s:46", "s:74", "s:83", "s:602", "s:605", "s:608" ], "params": {}, "parent": null, "results": [ "s:621" ] }, { "annotation": null, "id": "r:631", "op": "linear.sum_nodes", "operands": [ "s:623", "s:625", "s:627" ], "params": { "max_fanout": null }, "parent": "r:630", "results": [ "s:621" ] }, { "annotation": null, "id": "r:632", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:46", "s:602" ], "params": {}, "parent": "r:630", "results": [ "s:623" ] }, { "annotation": null, "id": "r:633", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:624" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:632", "results": [ "s:623" ] }, { "annotation": null, "id": "r:634", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:74", "s:605" ], "params": {}, "parent": "r:630", "results": [ "s:625" ] }, { "annotation": null, "id": "r:635", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:626" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:634", "results": [ "s:625" ] }, { "annotation": null, "id": "r:636", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:83", "s:608" ], "params": {}, "parent": "r:630", "results": [ "s:627" ] }, { "annotation": null, "id": "r:637", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:628" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:636", "results": [ "s:627" ] }, { "annotation": null, "id": "r:638", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:631", "s:637" ], "params": {}, "parent": "r:0", "results": [ "s:629" ] }, { "annotation": null, "id": "r:639", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:630" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:638", "results": [ "s:629" ] }, { "annotation": null, "id": "r:640", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:634", "s:635" ], "params": {}, "parent": "r:0", "results": [ "s:631" ] }, { "annotation": null, "id": "r:641", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:632" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:640", "results": [ "s:631" ] }, { "annotation": null, "id": "r:642", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:632" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:641", "results": [ "s:631" ] }, { "annotation": null, "id": "r:643", "op": "linear.sum_nodes", "operands": [ "s:634", "s:635" ], "params": { "max_fanout": null }, "parent": "r:640", "results": [ "s:632" ] }, { "annotation": null, "id": "r:644", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:18" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 2.5, "true_level": 1.0 }, "parent": "r:0", "results": [ "s:634" ] }, { "annotation": null, "id": "r:645", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:18" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:644", "results": [ "s:634" ] }, { "annotation": null, "id": "r:646", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:636" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -3.5, "true_level": 1.0 }, "parent": "r:0", "results": [ "s:635" ] }, { "annotation": null, "id": "r:647", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:636" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:646", "results": [ "s:635" ] }, { "annotation": null, "id": "r:648", "op": "linear.negate", "operands": [ "s:18" ], "params": {}, "parent": "r:0", "results": [ "s:636" ] }, { "annotation": null, "id": "r:649", "op": "swiglu.map_select.broadcast_select", "operands": [ "s:29", "s:640" ], "params": { "d_fill": 17, "n_slots": 12 }, "parent": "r:28", "results": [ "s:638" ] }, { "annotation": null, "id": "r:650", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:639" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 204 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 204, 216 ] } }, "name": "broadcast_select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 204 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 204, 204 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 204 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 204, 216 ] } } }, "parent": "r:649", "results": [ "s:638" ] }, { "annotation": null, "id": "r:651", "op": "linear.concat", "operands": [ "s:482", "s:542", "s:568", "s:598", "s:621", "s:621", "s:447", "s:482", "s:542", "s:568", "s:598", "s:621" ], "params": {}, "parent": "r:28", "results": [ "s:640" ] }, { "annotation": null, "id": "r:652", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:643", "s:649" ], "params": {}, "parent": "r:0", "results": [ "s:641" ] }, { "annotation": null, "id": "r:653", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:642" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:652", "results": [ "s:641" ] }, { "annotation": null, "id": "r:654", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:646", "s:647" ], "params": {}, "parent": "r:0", "results": [ "s:643" ] }, { "annotation": null, "id": "r:655", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:644" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:654", "results": [ "s:643" ] }, { "annotation": null, "id": "r:656", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:644" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:655", "results": [ "s:643" ] }, { "annotation": null, "id": "r:657", "op": "linear.sum_nodes", "operands": [ "s:646", "s:647" ], "params": { "max_fanout": null }, "parent": "r:654", "results": [ "s:644" ] }, { "annotation": null, "id": "r:658", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:18" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 3.5, "true_level": 1.0 }, "parent": "r:0", "results": [ "s:646" ] }, { "annotation": null, "id": "r:659", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:18" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:658", "results": [ "s:646" ] }, { "annotation": null, "id": "r:660", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:648" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -4.5, "true_level": 1.0 }, "parent": "r:0", "results": [ "s:647" ] }, { "annotation": null, "id": "r:661", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:648" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:660", "results": [ "s:647" ] }, { "annotation": null, "id": "r:662", "op": "linear.negate", "operands": [ "s:18" ], "params": {}, "parent": "r:0", "results": [ "s:648" ] }, { "annotation": null, "id": "r:663", "op": "swiglu.map_select.broadcast_select", "operands": [ "s:29", "s:652" ], "params": { "d_fill": 17, "n_slots": 12 }, "parent": "r:28", "results": [ "s:650" ] }, { "annotation": null, "id": "r:664", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:651" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 204 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 204, 216 ] } }, "name": "broadcast_select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 204 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 204, 204 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 204 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 204, 216 ] } } }, "parent": "r:663", "results": [ "s:650" ] }, { "annotation": null, "id": "r:665", "op": "linear.concat", "operands": [ "s:542", "s:568", "s:598", "s:621", "s:621", "s:621", "s:482", "s:542", "s:568", "s:598", "s:621", "s:621" ], "params": {}, "parent": "r:28", "results": [ "s:652" ] }, { "annotation": null, "id": "r:666", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:655", "s:661" ], "params": {}, "parent": "r:0", "results": [ "s:653" ] }, { "annotation": null, "id": "r:667", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:654" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:666", "results": [ "s:653" ] }, { "annotation": null, "id": "r:668", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:658", "s:659" ], "params": {}, "parent": "r:0", "results": [ "s:655" ] }, { "annotation": null, "id": "r:669", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:656" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:668", "results": [ "s:655" ] }, { "annotation": null, "id": "r:670", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:656" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:669", "results": [ "s:655" ] }, { "annotation": null, "id": "r:671", "op": "linear.sum_nodes", "operands": [ "s:658", "s:659" ], "params": { "max_fanout": null }, "parent": "r:668", "results": [ "s:656" ] }, { "annotation": null, "id": "r:672", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:18" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 4.5, "true_level": 1.0 }, "parent": "r:0", "results": [ "s:658" ] }, { "annotation": null, "id": "r:673", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:18" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:672", "results": [ "s:658" ] }, { "annotation": null, "id": "r:674", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:660" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -5.5, "true_level": 1.0 }, "parent": "r:0", "results": [ "s:659" ] }, { "annotation": null, "id": "r:675", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:660" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:674", "results": [ "s:659" ] }, { "annotation": null, "id": "r:676", "op": "linear.negate", "operands": [ "s:18" ], "params": {}, "parent": "r:0", "results": [ "s:660" ] }, { "annotation": null, "id": "r:677", "op": "swiglu.map_select.broadcast_select", "operands": [ "s:29", "s:664" ], "params": { "d_fill": 17, "n_slots": 12 }, "parent": "r:28", "results": [ "s:662" ] }, { "annotation": null, "id": "r:678", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:663" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 204 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 204, 216 ] } }, "name": "broadcast_select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 204 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 204, 204 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 204 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 204, 216 ] } } }, "parent": "r:677", "results": [ "s:662" ] }, { "annotation": null, "id": "r:679", "op": "linear.concat", "operands": [ "s:568", "s:598", "s:621", "s:621", "s:621", "s:621", "s:542", "s:568", "s:598", "s:621", "s:621", "s:621" ], "params": {}, "parent": "r:28", "results": [ "s:664" ] }, { "annotation": null, "id": "r:680", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:667", "s:673" ], "params": {}, "parent": "r:0", "results": [ "s:665" ] }, { "annotation": null, "id": "r:681", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:666" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:680", "results": [ "s:665" ] }, { "annotation": null, "id": "r:682", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:670", "s:671" ], "params": {}, "parent": "r:0", "results": [ "s:667" ] }, { "annotation": null, "id": "r:683", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:668" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:682", "results": [ "s:667" ] }, { "annotation": null, "id": "r:684", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:668" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:683", "results": [ "s:667" ] }, { "annotation": null, "id": "r:685", "op": "linear.sum_nodes", "operands": [ "s:670", "s:671" ], "params": { "max_fanout": null }, "parent": "r:682", "results": [ "s:668" ] }, { "annotation": null, "id": "r:686", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:18" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 5.5, "true_level": 1.0 }, "parent": "r:0", "results": [ "s:670" ] }, { "annotation": null, "id": "r:687", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:18" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:686", "results": [ "s:670" ] }, { "annotation": null, "id": "r:688", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:672" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -6.5, "true_level": 1.0 }, "parent": "r:0", "results": [ "s:671" ] }, { "annotation": null, "id": "r:689", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:672" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:688", "results": [ "s:671" ] }, { "annotation": null, "id": "r:690", "op": "linear.negate", "operands": [ "s:18" ], "params": {}, "parent": "r:0", "results": [ "s:672" ] }, { "annotation": null, "id": "r:691", "op": "swiglu.map_select.broadcast_select", "operands": [ "s:29", "s:676" ], "params": { "d_fill": 17, "n_slots": 12 }, "parent": "r:28", "results": [ "s:674" ] }, { "annotation": null, "id": "r:692", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:675" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 204 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 204, 216 ] } }, "name": "broadcast_select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 204 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 204, 204 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 204 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 204, 216 ] } } }, "parent": "r:691", "results": [ "s:674" ] }, { "annotation": null, "id": "r:693", "op": "linear.concat", "operands": [ "s:598", "s:621", "s:621", "s:621", "s:621", "s:621", "s:568", "s:598", "s:621", "s:621", "s:621", "s:621" ], "params": {}, "parent": "r:28", "results": [ "s:676" ] }, { "annotation": null, "id": "r:694", "op": "swiglu.logic_ops.cond_gate", "operands": [ "s:679", "s:685" ], "params": {}, "parent": "r:0", "results": [ "s:677" ] }, { "annotation": null, "id": "r:695", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:678" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } }, "name": "cond_gate", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 17 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 17 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 17, 18 ] } } }, "parent": "r:694", "results": [ "s:677" ] }, { "annotation": null, "id": "r:696", "op": "swiglu.logic_ops.bool_all_true", "operands": [ "s:682", "s:683" ], "params": {}, "parent": "r:0", "results": [ "s:679" ] }, { "annotation": null, "id": "r:697", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:680" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 1.0, "true_level": 1.0 }, "parent": "r:696", "results": [ "s:679" ] }, { "annotation": null, "id": "r:698", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:680" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:697", "results": [ "s:679" ] }, { "annotation": null, "id": "r:699", "op": "linear.sum_nodes", "operands": [ "s:682", "s:683" ], "params": { "max_fanout": null }, "parent": "r:696", "results": [ "s:680" ] }, { "annotation": null, "id": "r:700", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:18" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": 6.5, "true_level": 1.0 }, "parent": "r:0", "results": [ "s:682" ] }, { "annotation": null, "id": "r:701", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:18" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:700", "results": [ "s:682" ] }, { "annotation": null, "id": "r:702", "op": "swiglu.arithmetic_ops.compare", "operands": [ "s:684" ], "params": { "false_level": -1.0, "sharpness": null, "thresh": -7.5, "true_level": 1.0 }, "parent": "r:0", "results": [ "s:683" ] }, { "annotation": null, "id": "r:703", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:684" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 2 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "name": "compare", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 1 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 2, 1 ] } }, "up_bias": null, "up_proj": null }, "parent": "r:702", "results": [ "s:683" ] }, { "annotation": null, "id": "r:704", "op": "linear.negate", "operands": [ "s:18" ], "params": {}, "parent": "r:0", "results": [ "s:684" ] }, { "annotation": null, "id": "r:705", "op": "swiglu.map_select.broadcast_select", "operands": [ "s:29", "s:688" ], "params": { "d_fill": 17, "n_slots": 12 }, "parent": "r:28", "results": [ "s:686" ] }, { "annotation": null, "id": "r:706", "op": "swiglu.swiglu_ffn.swiglu_ffn", "operands": [ "s:687" ], "params": { "gate_bias": { "tensor": { "dtype": "float32", "shape": [ 204 ] } }, "gate_proj": { "tensor": { "dtype": "float32", "shape": [ 204, 216 ] } }, "name": "broadcast_select", "output_bias": { "tensor": { "dtype": "float32", "shape": [ 204 ] } }, "output_proj": { "tensor": { "dtype": "float32", "shape": [ 204, 204 ] } }, "up_bias": { "tensor": { "dtype": "float32", "shape": [ 204 ] } }, "up_proj": { "tensor": { "dtype": "float32", "shape": [ 204, 216 ] } } }, "parent": "r:705", "results": [ "s:686" ] }, { "annotation": null, "id": "r:707", "op": "linear.concat", "operands": [ "s:621", "s:621", "s:621", "s:621", "s:621", "s:621", "s:598", "s:621", "s:621", "s:621", "s:621", "s:621" ], "params": {}, "parent": "r:28", "results": [ "s:688" ] }, { "annotation": null, "id": "r:708", "op": "inout_nodes.create_literal_value", "operands": [], "params": { "name": "", "vector": { "tensor": { "dtype": "float32", "shape": [ 17 ] } } }, "parent": "r:0", "results": [ "s:689" ] } ], "sha256": "6f725947673e9902194428a90a7a18a6364f4c84f893419e4e6d6039f4dd3474" } }, "integrity": { "scope": "every manifest section except this one", "sha256": "c9d231ab4e43cf241333b68a2232b11ac652b6016edeba9fce85bc512fdc79f6" }, "intervention_contract": { "guarantee": "The manifest describes the unmodified compiled program. Interventions outside a node's stated value contract are not covered by torchwright's semantic or numerical guarantees.", "node_value_contracts": "graphs.source.nodes[*].value_contract", "physical_coordinates": { "attention_heads": "schedule.layers[*].attention_operations[*].heads", "mlp_neurons": "schedule.layers[*].mlp_operations[*].hidden_slots", "residual": "schedule.layers[*].attention_operations[*] and mlp_operations[*] target/source/query/key column runs" } }, "metadata": { "source": { "arguments": { "max_digits": 3 }, "entrypoint": "examples.calculator_advanced:create_network_parts" }, "task": { "example": "calculator_advanced", "max_digits": 3 } }, "model": { "activation": "swish", "active_attention_heads_per_layer": [ 1, 4, 3, 3, 8, 4, 2, 4, 5, 2, 4, 8, 12, 2, 2, 6, 4, 4, 4, 3, 4, 1, 1, 1, 2, 1, 1, 32, 6 ], "active_mlp_neurons_per_layer": [ 16, 29, 73, 25, 376, 364, 82, 54, 6, 1, 3, 1543, 399, 1888, 2254, 1274, 792, 696, 708, 446, 128, 113, 23, 9, 5, 47, 1963, 3415, 307 ], "architecture": "phi3", "bias": false, "d_head": 64, "d_model": 2048, "max_sequence_length": 512, "n_layers": 29, "rms_norm": true, "rms_norm_eps": 1e-05, "rope_base": 500000.0, "rotary_width": 64, "stored_attention_heads": 32, "stored_mlp_neurons": 3415 }, "observability": { "abstract_residual_states": [ { "hook": "blocks.0.hook_resid_pre", "state": "input" }, { "hook": "blocks.0.hook_resid_post", "state": "L0.post" }, { "hook": "blocks.1.hook_resid_post", "state": "L1.post" }, { "hook": "blocks.2.hook_resid_post", "state": "L2.post" }, { "hook": "blocks.3.hook_resid_post", "state": "L3.post" }, { "hook": "blocks.4.hook_resid_post", "state": "L4.post" }, { "hook": "blocks.5.hook_resid_post", "state": "L5.post" }, { "hook": "blocks.6.hook_resid_post", "state": "L6.post" }, { "hook": "blocks.7.hook_resid_post", "state": "L7.post" }, { "hook": "blocks.8.hook_resid_post", "state": "L8.post" }, { "hook": "blocks.9.hook_resid_post", "state": "L9.post" }, { "hook": "blocks.10.hook_resid_post", "state": "L10.post" }, { "hook": "blocks.11.hook_resid_post", "state": "L11.post" }, { "hook": "blocks.12.hook_resid_post", "state": "L12.post" }, { "hook": "blocks.13.hook_resid_post", "state": "L13.post" }, { "hook": "blocks.14.hook_resid_post", "state": "L14.post" }, { "hook": "blocks.15.hook_resid_post", "state": "L15.post" }, { "hook": "blocks.16.hook_resid_post", "state": "L16.post" }, { "hook": "blocks.17.hook_resid_post", "state": "L17.post" }, { "hook": "blocks.18.hook_resid_post", "state": "L18.post" }, { "hook": "blocks.19.hook_resid_post", "state": "L19.post" }, { "hook": "blocks.20.hook_resid_post", "state": "L20.post" }, { "hook": "blocks.21.hook_resid_post", "state": "L21.post" }, { "hook": "blocks.22.hook_resid_post", "state": "L22.post" }, { "hook": "blocks.23.hook_resid_post", "state": "L23.post" }, { "hook": "blocks.24.hook_resid_post", "state": "L24.post" }, { "hook": "blocks.25.hook_resid_post", "state": "L25.post" }, { "hook": "blocks.26.hook_resid_post", "state": "L26.post" }, { "hook": "blocks.27.hook_resid_post", "state": "L27.post" }, { "hook": "blocks.28.hook_resid_post", "state": "L28.post" } ], "transformer_lens": { "attention": { "head_axis": 2, "k": "blocks.{layer}.attn.hook_k", "pattern": "blocks.{layer}.attn.hook_pattern", "q": "blocks.{layer}.attn.hook_q", "result": "blocks.{layer}.attn.hook_result", "v": "blocks.{layer}.attn.hook_v" }, "mlp": { "neuron_axis": -1, "post": "blocks.{layer}.mlp.hook_post", "pre": "blocks.{layer}.mlp.hook_pre" }, "note": "Use schedule.layers head and hidden-slot ranges to slice the standard hooks. Hook availability depends on the adapter version.", "residual_states": [ { "hook": "blocks.0.hook_resid_pre", "state": "input" }, { "hook": "blocks.0.hook_resid_post", "state": "L0.post" }, { "hook": "blocks.1.hook_resid_post", "state": "L1.post" }, { "hook": "blocks.2.hook_resid_post", "state": "L2.post" }, { "hook": "blocks.3.hook_resid_post", "state": "L3.post" }, { "hook": "blocks.4.hook_resid_post", "state": "L4.post" }, { "hook": "blocks.5.hook_resid_post", "state": "L5.post" }, { "hook": "blocks.6.hook_resid_post", "state": "L6.post" }, { "hook": "blocks.7.hook_resid_post", "state": "L7.post" }, { "hook": "blocks.8.hook_resid_post", "state": "L8.post" }, { "hook": "blocks.9.hook_resid_post", "state": "L9.post" }, { "hook": "blocks.10.hook_resid_post", "state": "L10.post" }, { "hook": "blocks.11.hook_resid_post", "state": "L11.post" }, { "hook": "blocks.12.hook_resid_post", "state": "L12.post" }, { "hook": "blocks.13.hook_resid_post", "state": "L13.post" }, { "hook": "blocks.14.hook_resid_post", "state": "L14.post" }, { "hook": "blocks.15.hook_resid_post", "state": "L15.post" }, { "hook": "blocks.16.hook_resid_post", "state": "L16.post" }, { "hook": "blocks.17.hook_resid_post", "state": "L17.post" }, { "hook": "blocks.18.hook_resid_post", "state": "L18.post" }, { "hook": "blocks.19.hook_resid_post", "state": "L19.post" }, { "hook": "blocks.20.hook_resid_post", "state": "L20.post" }, { "hook": "blocks.21.hook_resid_post", "state": "L21.post" }, { "hook": "blocks.22.hook_resid_post", "state": "L22.post" }, { "hook": "blocks.23.hook_resid_post", "state": "L23.post" }, { "hook": "blocks.24.hook_resid_post", "state": "L24.post" }, { "hook": "blocks.25.hook_resid_post", "state": "L25.post" }, { "hook": "blocks.26.hook_resid_post", "state": "L26.post" }, { "hook": "blocks.27.hook_resid_post", "state": "L27.post" }, { "hook": "blocks.28.hook_resid_post", "state": "L28.post" } ] } }, "parameter_support": { "file": "torchwright_schematic_support.npz", "format": "torchwright.csr_support.v1", "sha256": "a1f439d473c33c5d6099ad815acc0dd18945e58fc49f06cc76d6cfc692098352", "tensors": { "model.embed_tokens.weight": { "chunks": [ { "indices": "tensor_000116_chunk_000000_indices", "indptr": "tensor_000116_chunk_000000_indptr", "nnz": 71, "row_count": 18, "row_start": 0 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 71, "shape": [ 18, 2048 ] }, "model.layers.0.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000118_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.0.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000003_chunk_000000_indices", "indptr": "tensor_000003_chunk_000000_indptr", "nnz": 39, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000003_chunk_000001_indices", "indptr": "tensor_000003_chunk_000001_indptr", "nnz": 0, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000003_chunk_000002_indices", "indptr": "tensor_000003_chunk_000002_indptr", "nnz": 0, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000003_chunk_000003_indices", "indptr": "tensor_000003_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000003_chunk_000004_indices", "indptr": "tensor_000003_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000003_chunk_000005_indices", "indptr": "tensor_000003_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000003_chunk_000006_indices", "indptr": "tensor_000003_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 39, "shape": [ 2048, 3415 ] }, "model.layers.0.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000002_chunk_000000_indices", "indptr": "tensor_000002_chunk_000000_indptr", "nnz": 16, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000002_chunk_000001_indices", "indptr": "tensor_000002_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000002_chunk_000002_indices", "indptr": "tensor_000002_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000002_chunk_000003_indices", "indptr": "tensor_000002_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000002_chunk_000004_indices", "indptr": "tensor_000002_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000002_chunk_000005_indices", "indptr": "tensor_000002_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000002_chunk_000006_indices", "indptr": "tensor_000002_chunk_000006_indptr", "nnz": 16, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000002_chunk_000007_indices", "indptr": "tensor_000002_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000002_chunk_000008_indices", "indptr": "tensor_000002_chunk_000008_indptr", "nnz": 0, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000002_chunk_000009_indices", "indptr": "tensor_000002_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000002_chunk_000010_indices", "indptr": "tensor_000002_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000002_chunk_000011_indices", "indptr": "tensor_000002_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000002_chunk_000012_indices", "indptr": "tensor_000002_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000002_chunk_000013_indices", "indptr": "tensor_000002_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 32, "shape": [ 6830, 2048 ] }, "model.layers.0.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000119_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.0.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000001_chunk_000000_indices", "indptr": "tensor_000001_chunk_000000_indptr", "nnz": 0, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000001_chunk_000001_indices", "indptr": "tensor_000001_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000001_chunk_000002_indices", "indptr": "tensor_000001_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000001_chunk_000003_indices", "indptr": "tensor_000001_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 0, "shape": [ 2048, 2048 ] }, "model.layers.0.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000000_chunk_000000_indices", "indptr": "tensor_000000_chunk_000000_indptr", "nnz": 0, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000000_chunk_000001_indices", "indptr": "tensor_000000_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000000_chunk_000002_indices", "indptr": "tensor_000000_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000000_chunk_000003_indices", "indptr": "tensor_000000_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000000_chunk_000004_indices", "indptr": "tensor_000000_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000000_chunk_000005_indices", "indptr": "tensor_000000_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000000_chunk_000006_indices", "indptr": "tensor_000000_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000000_chunk_000007_indices", "indptr": "tensor_000000_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000000_chunk_000008_indices", "indptr": "tensor_000000_chunk_000008_indptr", "nnz": 0, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000000_chunk_000009_indices", "indptr": "tensor_000000_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000000_chunk_000010_indices", "indptr": "tensor_000000_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000000_chunk_000011_indices", "indptr": "tensor_000000_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 0, "shape": [ 6144, 2048 ] }, "model.layers.1.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000120_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.1.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000007_chunk_000000_indices", "indptr": "tensor_000007_chunk_000000_indptr", "nnz": 33, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000007_chunk_000001_indices", "indptr": "tensor_000007_chunk_000001_indptr", "nnz": 0, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000007_chunk_000002_indices", "indptr": "tensor_000007_chunk_000002_indptr", "nnz": 0, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000007_chunk_000003_indices", "indptr": "tensor_000007_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000007_chunk_000004_indices", "indptr": "tensor_000007_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000007_chunk_000005_indices", "indptr": "tensor_000007_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000007_chunk_000006_indices", "indptr": "tensor_000007_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 33, "shape": [ 2048, 3415 ] }, "model.layers.1.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000006_chunk_000000_indices", "indptr": "tensor_000006_chunk_000000_indptr", "nnz": 34, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000006_chunk_000001_indices", "indptr": "tensor_000006_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000006_chunk_000002_indices", "indptr": "tensor_000006_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000006_chunk_000003_indices", "indptr": "tensor_000006_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000006_chunk_000004_indices", "indptr": "tensor_000006_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000006_chunk_000005_indices", "indptr": "tensor_000006_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000006_chunk_000006_indices", "indptr": "tensor_000006_chunk_000006_indptr", "nnz": 29, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000006_chunk_000007_indices", "indptr": "tensor_000006_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000006_chunk_000008_indices", "indptr": "tensor_000006_chunk_000008_indptr", "nnz": 0, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000006_chunk_000009_indices", "indptr": "tensor_000006_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000006_chunk_000010_indices", "indptr": "tensor_000006_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000006_chunk_000011_indices", "indptr": "tensor_000006_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000006_chunk_000012_indices", "indptr": "tensor_000006_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000006_chunk_000013_indices", "indptr": "tensor_000006_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 63, "shape": [ 6830, 2048 ] }, "model.layers.1.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000121_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.1.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000005_chunk_000000_indices", "indptr": "tensor_000005_chunk_000000_indptr", "nnz": 12, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000005_chunk_000001_indices", "indptr": "tensor_000005_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000005_chunk_000002_indices", "indptr": "tensor_000005_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000005_chunk_000003_indices", "indptr": "tensor_000005_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 12, "shape": [ 2048, 2048 ] }, "model.layers.1.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000004_chunk_000000_indices", "indptr": "tensor_000004_chunk_000000_indptr", "nnz": 97, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000004_chunk_000001_indices", "indptr": "tensor_000004_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000004_chunk_000002_indices", "indptr": "tensor_000004_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000004_chunk_000003_indices", "indptr": "tensor_000004_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000004_chunk_000004_indices", "indptr": "tensor_000004_chunk_000004_indptr", "nnz": 97, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000004_chunk_000005_indices", "indptr": "tensor_000004_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000004_chunk_000006_indices", "indptr": "tensor_000004_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000004_chunk_000007_indices", "indptr": "tensor_000004_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000004_chunk_000008_indices", "indptr": "tensor_000004_chunk_000008_indptr", "nnz": 12, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000004_chunk_000009_indices", "indptr": "tensor_000004_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000004_chunk_000010_indices", "indptr": "tensor_000004_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000004_chunk_000011_indices", "indptr": "tensor_000004_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 206, "shape": [ 6144, 2048 ] }, "model.layers.10.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000138_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.10.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000043_chunk_000000_indices", "indptr": "tensor_000043_chunk_000000_indptr", "nnz": 19, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000043_chunk_000001_indices", "indptr": "tensor_000043_chunk_000001_indptr", "nnz": 0, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000043_chunk_000002_indices", "indptr": "tensor_000043_chunk_000002_indptr", "nnz": 0, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000043_chunk_000003_indices", "indptr": "tensor_000043_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000043_chunk_000004_indices", "indptr": "tensor_000043_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000043_chunk_000005_indices", "indptr": "tensor_000043_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000043_chunk_000006_indices", "indptr": "tensor_000043_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 19, "shape": [ 2048, 3415 ] }, "model.layers.10.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000042_chunk_000000_indices", "indptr": "tensor_000042_chunk_000000_indptr", "nnz": 3, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000042_chunk_000001_indices", "indptr": "tensor_000042_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000042_chunk_000002_indices", "indptr": "tensor_000042_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000042_chunk_000003_indices", "indptr": "tensor_000042_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000042_chunk_000004_indices", "indptr": "tensor_000042_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000042_chunk_000005_indices", "indptr": "tensor_000042_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000042_chunk_000006_indices", "indptr": "tensor_000042_chunk_000006_indptr", "nnz": 3, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000042_chunk_000007_indices", "indptr": "tensor_000042_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000042_chunk_000008_indices", "indptr": "tensor_000042_chunk_000008_indptr", "nnz": 0, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000042_chunk_000009_indices", "indptr": "tensor_000042_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000042_chunk_000010_indices", "indptr": "tensor_000042_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000042_chunk_000011_indices", "indptr": "tensor_000042_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000042_chunk_000012_indices", "indptr": "tensor_000042_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000042_chunk_000013_indices", "indptr": "tensor_000042_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 6, "shape": [ 6830, 2048 ] }, "model.layers.10.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000139_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.10.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000041_chunk_000000_indices", "indptr": "tensor_000041_chunk_000000_indptr", "nnz": 87, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000041_chunk_000001_indices", "indptr": "tensor_000041_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000041_chunk_000002_indices", "indptr": "tensor_000041_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000041_chunk_000003_indices", "indptr": "tensor_000041_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 87, "shape": [ 2048, 2048 ] }, "model.layers.10.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000040_chunk_000000_indices", "indptr": "tensor_000040_chunk_000000_indptr", "nnz": 76, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000040_chunk_000001_indices", "indptr": "tensor_000040_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000040_chunk_000002_indices", "indptr": "tensor_000040_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000040_chunk_000003_indices", "indptr": "tensor_000040_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000040_chunk_000004_indices", "indptr": "tensor_000040_chunk_000004_indptr", "nnz": 76, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000040_chunk_000005_indices", "indptr": "tensor_000040_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000040_chunk_000006_indices", "indptr": "tensor_000040_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000040_chunk_000007_indices", "indptr": "tensor_000040_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000040_chunk_000008_indices", "indptr": "tensor_000040_chunk_000008_indptr", "nnz": 87, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000040_chunk_000009_indices", "indptr": "tensor_000040_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000040_chunk_000010_indices", "indptr": "tensor_000040_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000040_chunk_000011_indices", "indptr": "tensor_000040_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 239, "shape": [ 6144, 2048 ] }, "model.layers.11.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000140_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.11.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000047_chunk_000000_indices", "indptr": "tensor_000047_chunk_000000_indptr", "nnz": 304, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000047_chunk_000001_indices", "indptr": "tensor_000047_chunk_000001_indptr", "nnz": 2717, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000047_chunk_000002_indices", "indptr": "tensor_000047_chunk_000002_indptr", "nnz": 0, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000047_chunk_000003_indices", "indptr": "tensor_000047_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000047_chunk_000004_indices", "indptr": "tensor_000047_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000047_chunk_000005_indices", "indptr": "tensor_000047_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000047_chunk_000006_indices", "indptr": "tensor_000047_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 3021, "shape": [ 2048, 3415 ] }, "model.layers.11.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000046_chunk_000000_indices", "indptr": "tensor_000046_chunk_000000_indptr", "nnz": 1534, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000046_chunk_000001_indices", "indptr": "tensor_000046_chunk_000001_indptr", "nnz": 1536, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000046_chunk_000002_indices", "indptr": "tensor_000046_chunk_000002_indptr", "nnz": 1466, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000046_chunk_000003_indices", "indptr": "tensor_000046_chunk_000003_indptr", "nnz": 7, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000046_chunk_000004_indices", "indptr": "tensor_000046_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000046_chunk_000005_indices", "indptr": "tensor_000046_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000046_chunk_000006_indices", "indptr": "tensor_000046_chunk_000006_indptr", "nnz": 169, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000046_chunk_000007_indices", "indptr": "tensor_000046_chunk_000007_indptr", "nnz": 512, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000046_chunk_000008_indices", "indptr": "tensor_000046_chunk_000008_indptr", "nnz": 512, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000046_chunk_000009_indices", "indptr": "tensor_000046_chunk_000009_indptr", "nnz": 350, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000046_chunk_000010_indices", "indptr": "tensor_000046_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000046_chunk_000011_indices", "indptr": "tensor_000046_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000046_chunk_000012_indices", "indptr": "tensor_000046_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000046_chunk_000013_indices", "indptr": "tensor_000046_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 6086, "shape": [ 6830, 2048 ] }, "model.layers.11.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000141_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.11.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000045_chunk_000000_indices", "indptr": "tensor_000045_chunk_000000_indptr", "nnz": 202, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000045_chunk_000001_indices", "indptr": "tensor_000045_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000045_chunk_000002_indices", "indptr": "tensor_000045_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000045_chunk_000003_indices", "indptr": "tensor_000045_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 202, "shape": [ 2048, 2048 ] }, "model.layers.11.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000044_chunk_000000_indices", "indptr": "tensor_000044_chunk_000000_indptr", "nnz": 194, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000044_chunk_000001_indices", "indptr": "tensor_000044_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000044_chunk_000002_indices", "indptr": "tensor_000044_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000044_chunk_000003_indices", "indptr": "tensor_000044_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000044_chunk_000004_indices", "indptr": "tensor_000044_chunk_000004_indptr", "nnz": 194, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000044_chunk_000005_indices", "indptr": "tensor_000044_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000044_chunk_000006_indices", "indptr": "tensor_000044_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000044_chunk_000007_indices", "indptr": "tensor_000044_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000044_chunk_000008_indices", "indptr": "tensor_000044_chunk_000008_indptr", "nnz": 202, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000044_chunk_000009_indices", "indptr": "tensor_000044_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000044_chunk_000010_indices", "indptr": "tensor_000044_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000044_chunk_000011_indices", "indptr": "tensor_000044_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 590, "shape": [ 6144, 2048 ] }, "model.layers.12.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000142_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.12.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000051_chunk_000000_indices", "indptr": "tensor_000051_chunk_000000_indptr", "nnz": 88, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000051_chunk_000001_indices", "indptr": "tensor_000051_chunk_000001_indptr", "nnz": 319, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000051_chunk_000002_indices", "indptr": "tensor_000051_chunk_000002_indptr", "nnz": 0, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000051_chunk_000003_indices", "indptr": "tensor_000051_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000051_chunk_000004_indices", "indptr": "tensor_000051_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000051_chunk_000005_indices", "indptr": "tensor_000051_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000051_chunk_000006_indices", "indptr": "tensor_000051_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 407, "shape": [ 2048, 3415 ] }, "model.layers.12.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000050_chunk_000000_indices", "indptr": "tensor_000050_chunk_000000_indptr", "nnz": 517, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000050_chunk_000001_indices", "indptr": "tensor_000050_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000050_chunk_000002_indices", "indptr": "tensor_000050_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000050_chunk_000003_indices", "indptr": "tensor_000050_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000050_chunk_000004_indices", "indptr": "tensor_000050_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000050_chunk_000005_indices", "indptr": "tensor_000050_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000050_chunk_000006_indices", "indptr": "tensor_000050_chunk_000006_indptr", "nnz": 169, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000050_chunk_000007_indices", "indptr": "tensor_000050_chunk_000007_indptr", "nnz": 230, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000050_chunk_000008_indices", "indptr": "tensor_000050_chunk_000008_indptr", "nnz": 0, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000050_chunk_000009_indices", "indptr": "tensor_000050_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000050_chunk_000010_indices", "indptr": "tensor_000050_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000050_chunk_000011_indices", "indptr": "tensor_000050_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000050_chunk_000012_indices", "indptr": "tensor_000050_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000050_chunk_000013_indices", "indptr": "tensor_000050_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 916, "shape": [ 6830, 2048 ] }, "model.layers.12.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000143_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.12.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000049_chunk_000000_indices", "indptr": "tensor_000049_chunk_000000_indptr", "nnz": 228, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000049_chunk_000001_indices", "indptr": "tensor_000049_chunk_000001_indptr", "nnz": 34, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000049_chunk_000002_indices", "indptr": "tensor_000049_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000049_chunk_000003_indices", "indptr": "tensor_000049_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 262, "shape": [ 2048, 2048 ] }, "model.layers.12.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000048_chunk_000000_indices", "indptr": "tensor_000048_chunk_000000_indptr", "nnz": 512, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000048_chunk_000001_indices", "indptr": "tensor_000048_chunk_000001_indptr", "nnz": 256, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000048_chunk_000002_indices", "indptr": "tensor_000048_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000048_chunk_000003_indices", "indptr": "tensor_000048_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000048_chunk_000004_indices", "indptr": "tensor_000048_chunk_000004_indptr", "nnz": 512, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000048_chunk_000005_indices", "indptr": "tensor_000048_chunk_000005_indptr", "nnz": 256, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000048_chunk_000006_indices", "indptr": "tensor_000048_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000048_chunk_000007_indices", "indptr": "tensor_000048_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000048_chunk_000008_indices", "indptr": "tensor_000048_chunk_000008_indptr", "nnz": 438, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000048_chunk_000009_indices", "indptr": "tensor_000048_chunk_000009_indptr", "nnz": 174, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000048_chunk_000010_indices", "indptr": "tensor_000048_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000048_chunk_000011_indices", "indptr": "tensor_000048_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 2148, "shape": [ 6144, 2048 ] }, "model.layers.13.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000144_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.13.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000055_chunk_000000_indices", "indptr": "tensor_000055_chunk_000000_indptr", "nnz": 184, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000055_chunk_000001_indices", "indptr": "tensor_000055_chunk_000001_indptr", "nnz": 4071, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000055_chunk_000002_indices", "indptr": "tensor_000055_chunk_000002_indptr", "nnz": 0, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000055_chunk_000003_indices", "indptr": "tensor_000055_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000055_chunk_000004_indices", "indptr": "tensor_000055_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000055_chunk_000005_indices", "indptr": "tensor_000055_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000055_chunk_000006_indices", "indptr": "tensor_000055_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 4255, "shape": [ 2048, 3415 ] }, "model.layers.13.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000054_chunk_000000_indices", "indptr": "tensor_000054_chunk_000000_indptr", "nnz": 1022, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000054_chunk_000001_indices", "indptr": "tensor_000054_chunk_000001_indptr", "nnz": 1022, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000054_chunk_000002_indices", "indptr": "tensor_000054_chunk_000002_indptr", "nnz": 1211, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000054_chunk_000003_indices", "indptr": "tensor_000054_chunk_000003_indptr", "nnz": 1284, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000054_chunk_000004_indices", "indptr": "tensor_000054_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000054_chunk_000005_indices", "indptr": "tensor_000054_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000054_chunk_000006_indices", "indptr": "tensor_000054_chunk_000006_indptr", "nnz": 169, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000054_chunk_000007_indices", "indptr": "tensor_000054_chunk_000007_indptr", "nnz": 512, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000054_chunk_000008_indices", "indptr": "tensor_000054_chunk_000008_indptr", "nnz": 512, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000054_chunk_000009_indices", "indptr": "tensor_000054_chunk_000009_indptr", "nnz": 512, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000054_chunk_000010_indices", "indptr": "tensor_000054_chunk_000010_indptr", "nnz": 183, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000054_chunk_000011_indices", "indptr": "tensor_000054_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000054_chunk_000012_indices", "indptr": "tensor_000054_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000054_chunk_000013_indices", "indptr": "tensor_000054_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 6427, "shape": [ 6830, 2048 ] }, "model.layers.13.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000145_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.13.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000053_chunk_000000_indices", "indptr": "tensor_000053_chunk_000000_indptr", "nnz": 9, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000053_chunk_000001_indices", "indptr": "tensor_000053_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000053_chunk_000002_indices", "indptr": "tensor_000053_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000053_chunk_000003_indices", "indptr": "tensor_000053_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 9, "shape": [ 2048, 2048 ] }, "model.layers.13.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000052_chunk_000000_indices", "indptr": "tensor_000052_chunk_000000_indptr", "nnz": 128, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000052_chunk_000001_indices", "indptr": "tensor_000052_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000052_chunk_000002_indices", "indptr": "tensor_000052_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000052_chunk_000003_indices", "indptr": "tensor_000052_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000052_chunk_000004_indices", "indptr": "tensor_000052_chunk_000004_indptr", "nnz": 128, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000052_chunk_000005_indices", "indptr": "tensor_000052_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000052_chunk_000006_indices", "indptr": "tensor_000052_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000052_chunk_000007_indices", "indptr": "tensor_000052_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000052_chunk_000008_indices", "indptr": "tensor_000052_chunk_000008_indptr", "nnz": 9, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000052_chunk_000009_indices", "indptr": "tensor_000052_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000052_chunk_000010_indices", "indptr": "tensor_000052_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000052_chunk_000011_indices", "indptr": "tensor_000052_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 265, "shape": [ 6144, 2048 ] }, "model.layers.14.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000146_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.14.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000059_chunk_000000_indices", "indptr": "tensor_000059_chunk_000000_indptr", "nnz": 1388, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000059_chunk_000001_indices", "indptr": "tensor_000059_chunk_000001_indptr", "nnz": 1713, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000059_chunk_000002_indices", "indptr": "tensor_000059_chunk_000002_indptr", "nnz": 1199, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000059_chunk_000003_indices", "indptr": "tensor_000059_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000059_chunk_000004_indices", "indptr": "tensor_000059_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000059_chunk_000005_indices", "indptr": "tensor_000059_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000059_chunk_000006_indices", "indptr": "tensor_000059_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 4300, "shape": [ 2048, 3415 ] }, "model.layers.14.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000058_chunk_000000_indices", "indptr": "tensor_000058_chunk_000000_indptr", "nnz": 1020, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000058_chunk_000001_indices", "indptr": "tensor_000058_chunk_000001_indptr", "nnz": 1047, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000058_chunk_000002_indices", "indptr": "tensor_000058_chunk_000002_indptr", "nnz": 1649, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000058_chunk_000003_indices", "indptr": "tensor_000058_chunk_000003_indptr", "nnz": 556, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000058_chunk_000004_indices", "indptr": "tensor_000058_chunk_000004_indptr", "nnz": 206, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000058_chunk_000005_indices", "indptr": "tensor_000058_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000058_chunk_000006_indices", "indptr": "tensor_000058_chunk_000006_indptr", "nnz": 169, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000058_chunk_000007_indices", "indptr": "tensor_000058_chunk_000007_indptr", "nnz": 512, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000058_chunk_000008_indices", "indptr": "tensor_000058_chunk_000008_indptr", "nnz": 512, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000058_chunk_000009_indices", "indptr": "tensor_000058_chunk_000009_indptr", "nnz": 512, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000058_chunk_000010_indices", "indptr": "tensor_000058_chunk_000010_indptr", "nnz": 512, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000058_chunk_000011_indices", "indptr": "tensor_000058_chunk_000011_indptr", "nnz": 37, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000058_chunk_000012_indices", "indptr": "tensor_000058_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000058_chunk_000013_indices", "indptr": "tensor_000058_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 6732, "shape": [ 6830, 2048 ] }, "model.layers.14.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000147_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.14.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000057_chunk_000000_indices", "indptr": "tensor_000057_chunk_000000_indptr", "nnz": 5, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000057_chunk_000001_indices", "indptr": "tensor_000057_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000057_chunk_000002_indices", "indptr": "tensor_000057_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000057_chunk_000003_indices", "indptr": "tensor_000057_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 5, "shape": [ 2048, 2048 ] }, "model.layers.14.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000056_chunk_000000_indices", "indptr": "tensor_000056_chunk_000000_indptr", "nnz": 128, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000056_chunk_000001_indices", "indptr": "tensor_000056_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000056_chunk_000002_indices", "indptr": "tensor_000056_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000056_chunk_000003_indices", "indptr": "tensor_000056_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000056_chunk_000004_indices", "indptr": "tensor_000056_chunk_000004_indptr", "nnz": 128, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000056_chunk_000005_indices", "indptr": "tensor_000056_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000056_chunk_000006_indices", "indptr": "tensor_000056_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000056_chunk_000007_indices", "indptr": "tensor_000056_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000056_chunk_000008_indices", "indptr": "tensor_000056_chunk_000008_indptr", "nnz": 5, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000056_chunk_000009_indices", "indptr": "tensor_000056_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000056_chunk_000010_indices", "indptr": "tensor_000056_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000056_chunk_000011_indices", "indptr": "tensor_000056_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 261, "shape": [ 6144, 2048 ] }, "model.layers.15.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000148_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.15.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000063_chunk_000000_indices", "indptr": "tensor_000063_chunk_000000_indptr", "nnz": 1216, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000063_chunk_000001_indices", "indptr": "tensor_000063_chunk_000001_indptr", "nnz": 282, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000063_chunk_000002_indices", "indptr": "tensor_000063_chunk_000002_indptr", "nnz": 164, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000063_chunk_000003_indices", "indptr": "tensor_000063_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000063_chunk_000004_indices", "indptr": "tensor_000063_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000063_chunk_000005_indices", "indptr": "tensor_000063_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000063_chunk_000006_indices", "indptr": "tensor_000063_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 1662, "shape": [ 2048, 3415 ] }, "model.layers.15.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000062_chunk_000000_indices", "indptr": "tensor_000062_chunk_000000_indptr", "nnz": 1534, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000062_chunk_000001_indices", "indptr": "tensor_000062_chunk_000001_indptr", "nnz": 1806, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000062_chunk_000002_indices", "indptr": "tensor_000062_chunk_000002_indptr", "nnz": 338, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000062_chunk_000003_indices", "indptr": "tensor_000062_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000062_chunk_000004_indices", "indptr": "tensor_000062_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000062_chunk_000005_indices", "indptr": "tensor_000062_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000062_chunk_000006_indices", "indptr": "tensor_000062_chunk_000006_indptr", "nnz": 169, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000062_chunk_000007_indices", "indptr": "tensor_000062_chunk_000007_indptr", "nnz": 512, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000062_chunk_000008_indices", "indptr": "tensor_000062_chunk_000008_indptr", "nnz": 512, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000062_chunk_000009_indices", "indptr": "tensor_000062_chunk_000009_indptr", "nnz": 81, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000062_chunk_000010_indices", "indptr": "tensor_000062_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000062_chunk_000011_indices", "indptr": "tensor_000062_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000062_chunk_000012_indices", "indptr": "tensor_000062_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000062_chunk_000013_indices", "indptr": "tensor_000062_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 4952, "shape": [ 6830, 2048 ] }, "model.layers.15.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000149_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.15.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000061_chunk_000000_indices", "indptr": "tensor_000061_chunk_000000_indptr", "nnz": 202, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000061_chunk_000001_indices", "indptr": "tensor_000061_chunk_000001_indptr", "nnz": 100, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000061_chunk_000002_indices", "indptr": "tensor_000061_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000061_chunk_000003_indices", "indptr": "tensor_000061_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 302, "shape": [ 2048, 2048 ] }, "model.layers.15.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000060_chunk_000000_indices", "indptr": "tensor_000060_chunk_000000_indptr", "nnz": 384, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000060_chunk_000001_indices", "indptr": "tensor_000060_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000060_chunk_000002_indices", "indptr": "tensor_000060_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000060_chunk_000003_indices", "indptr": "tensor_000060_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000060_chunk_000004_indices", "indptr": "tensor_000060_chunk_000004_indptr", "nnz": 384, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000060_chunk_000005_indices", "indptr": "tensor_000060_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000060_chunk_000006_indices", "indptr": "tensor_000060_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000060_chunk_000007_indices", "indptr": "tensor_000060_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000060_chunk_000008_indices", "indptr": "tensor_000060_chunk_000008_indptr", "nnz": 302, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000060_chunk_000009_indices", "indptr": "tensor_000060_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000060_chunk_000010_indices", "indptr": "tensor_000060_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000060_chunk_000011_indices", "indptr": "tensor_000060_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 1070, "shape": [ 6144, 2048 ] }, "model.layers.16.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000150_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.16.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000067_chunk_000000_indices", "indptr": "tensor_000067_chunk_000000_indptr", "nnz": 554, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000067_chunk_000001_indices", "indptr": "tensor_000067_chunk_000001_indptr", "nnz": 606, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000067_chunk_000002_indices", "indptr": "tensor_000067_chunk_000002_indptr", "nnz": 34, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000067_chunk_000003_indices", "indptr": "tensor_000067_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000067_chunk_000004_indices", "indptr": "tensor_000067_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000067_chunk_000005_indices", "indptr": "tensor_000067_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000067_chunk_000006_indices", "indptr": "tensor_000067_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 1194, "shape": [ 2048, 3415 ] }, "model.layers.16.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000066_chunk_000000_indices", "indptr": "tensor_000066_chunk_000000_indptr", "nnz": 1843, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000066_chunk_000001_indices", "indptr": "tensor_000066_chunk_000001_indptr", "nnz": 280, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000066_chunk_000002_indices", "indptr": "tensor_000066_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000066_chunk_000003_indices", "indptr": "tensor_000066_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000066_chunk_000004_indices", "indptr": "tensor_000066_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000066_chunk_000005_indices", "indptr": "tensor_000066_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000066_chunk_000006_indices", "indptr": "tensor_000066_chunk_000006_indptr", "nnz": 169, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000066_chunk_000007_indices", "indptr": "tensor_000066_chunk_000007_indptr", "nnz": 512, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000066_chunk_000008_indices", "indptr": "tensor_000066_chunk_000008_indptr", "nnz": 111, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000066_chunk_000009_indices", "indptr": "tensor_000066_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000066_chunk_000010_indices", "indptr": "tensor_000066_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000066_chunk_000011_indices", "indptr": "tensor_000066_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000066_chunk_000012_indices", "indptr": "tensor_000066_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000066_chunk_000013_indices", "indptr": "tensor_000066_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 2915, "shape": [ 6830, 2048 ] }, "model.layers.16.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000151_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.16.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000065_chunk_000000_indices", "indptr": "tensor_000065_chunk_000000_indptr", "nnz": 8, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000065_chunk_000001_indices", "indptr": "tensor_000065_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000065_chunk_000002_indices", "indptr": "tensor_000065_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000065_chunk_000003_indices", "indptr": "tensor_000065_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 8, "shape": [ 2048, 2048 ] }, "model.layers.16.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000064_chunk_000000_indices", "indptr": "tensor_000064_chunk_000000_indptr", "nnz": 256, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000064_chunk_000001_indices", "indptr": "tensor_000064_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000064_chunk_000002_indices", "indptr": "tensor_000064_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000064_chunk_000003_indices", "indptr": "tensor_000064_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000064_chunk_000004_indices", "indptr": "tensor_000064_chunk_000004_indptr", "nnz": 256, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000064_chunk_000005_indices", "indptr": "tensor_000064_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000064_chunk_000006_indices", "indptr": "tensor_000064_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000064_chunk_000007_indices", "indptr": "tensor_000064_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000064_chunk_000008_indices", "indptr": "tensor_000064_chunk_000008_indptr", "nnz": 8, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000064_chunk_000009_indices", "indptr": "tensor_000064_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000064_chunk_000010_indices", "indptr": "tensor_000064_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000064_chunk_000011_indices", "indptr": "tensor_000064_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 520, "shape": [ 6144, 2048 ] }, "model.layers.17.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000152_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.17.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000071_chunk_000000_indices", "indptr": "tensor_000071_chunk_000000_indptr", "nnz": 767, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000071_chunk_000001_indices", "indptr": "tensor_000071_chunk_000001_indptr", "nnz": 188, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000071_chunk_000002_indices", "indptr": "tensor_000071_chunk_000002_indptr", "nnz": 68, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000071_chunk_000003_indices", "indptr": "tensor_000071_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000071_chunk_000004_indices", "indptr": "tensor_000071_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000071_chunk_000005_indices", "indptr": "tensor_000071_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000071_chunk_000006_indices", "indptr": "tensor_000071_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 1023, "shape": [ 2048, 3415 ] }, "model.layers.17.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000070_chunk_000000_indices", "indptr": "tensor_000070_chunk_000000_indptr", "nnz": 1766, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000070_chunk_000001_indices", "indptr": "tensor_000070_chunk_000001_indptr", "nnz": 184, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000070_chunk_000002_indices", "indptr": "tensor_000070_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000070_chunk_000003_indices", "indptr": "tensor_000070_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000070_chunk_000004_indices", "indptr": "tensor_000070_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000070_chunk_000005_indices", "indptr": "tensor_000070_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000070_chunk_000006_indices", "indptr": "tensor_000070_chunk_000006_indptr", "nnz": 169, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000070_chunk_000007_indices", "indptr": "tensor_000070_chunk_000007_indptr", "nnz": 512, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000070_chunk_000008_indices", "indptr": "tensor_000070_chunk_000008_indptr", "nnz": 15, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000070_chunk_000009_indices", "indptr": "tensor_000070_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000070_chunk_000010_indices", "indptr": "tensor_000070_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000070_chunk_000011_indices", "indptr": "tensor_000070_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000070_chunk_000012_indices", "indptr": "tensor_000070_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000070_chunk_000013_indices", "indptr": "tensor_000070_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 2646, "shape": [ 6830, 2048 ] }, "model.layers.17.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000153_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.17.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000069_chunk_000000_indices", "indptr": "tensor_000069_chunk_000000_indptr", "nnz": 78, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000069_chunk_000001_indices", "indptr": "tensor_000069_chunk_000001_indptr", "nnz": 37, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000069_chunk_000002_indices", "indptr": "tensor_000069_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000069_chunk_000003_indices", "indptr": "tensor_000069_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 115, "shape": [ 2048, 2048 ] }, "model.layers.17.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000068_chunk_000000_indices", "indptr": "tensor_000068_chunk_000000_indptr", "nnz": 256, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000068_chunk_000001_indices", "indptr": "tensor_000068_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000068_chunk_000002_indices", "indptr": "tensor_000068_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000068_chunk_000003_indices", "indptr": "tensor_000068_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000068_chunk_000004_indices", "indptr": "tensor_000068_chunk_000004_indptr", "nnz": 256, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000068_chunk_000005_indices", "indptr": "tensor_000068_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000068_chunk_000006_indices", "indptr": "tensor_000068_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000068_chunk_000007_indices", "indptr": "tensor_000068_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000068_chunk_000008_indices", "indptr": "tensor_000068_chunk_000008_indptr", "nnz": 115, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000068_chunk_000009_indices", "indptr": "tensor_000068_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000068_chunk_000010_indices", "indptr": "tensor_000068_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000068_chunk_000011_indices", "indptr": "tensor_000068_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 627, "shape": [ 6144, 2048 ] }, "model.layers.18.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000154_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.18.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000075_chunk_000000_indices", "indptr": "tensor_000075_chunk_000000_indptr", "nnz": 392, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000075_chunk_000001_indices", "indptr": "tensor_000075_chunk_000001_indptr", "nnz": 639, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000075_chunk_000002_indices", "indptr": "tensor_000075_chunk_000002_indptr", "nnz": 0, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000075_chunk_000003_indices", "indptr": "tensor_000075_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000075_chunk_000004_indices", "indptr": "tensor_000075_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000075_chunk_000005_indices", "indptr": "tensor_000075_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000075_chunk_000006_indices", "indptr": "tensor_000075_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 1031, "shape": [ 2048, 3415 ] }, "model.layers.18.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000074_chunk_000000_indices", "indptr": "tensor_000074_chunk_000000_indptr", "nnz": 1730, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000074_chunk_000001_indices", "indptr": "tensor_000074_chunk_000001_indptr", "nnz": 196, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000074_chunk_000002_indices", "indptr": "tensor_000074_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000074_chunk_000003_indices", "indptr": "tensor_000074_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000074_chunk_000004_indices", "indptr": "tensor_000074_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000074_chunk_000005_indices", "indptr": "tensor_000074_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000074_chunk_000006_indices", "indptr": "tensor_000074_chunk_000006_indptr", "nnz": 169, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000074_chunk_000007_indices", "indptr": "tensor_000074_chunk_000007_indptr", "nnz": 512, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000074_chunk_000008_indices", "indptr": "tensor_000074_chunk_000008_indptr", "nnz": 27, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000074_chunk_000009_indices", "indptr": "tensor_000074_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000074_chunk_000010_indices", "indptr": "tensor_000074_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000074_chunk_000011_indices", "indptr": "tensor_000074_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000074_chunk_000012_indices", "indptr": "tensor_000074_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000074_chunk_000013_indices", "indptr": "tensor_000074_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 2634, "shape": [ 6830, 2048 ] }, "model.layers.18.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000155_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.18.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000073_chunk_000000_indices", "indptr": "tensor_000073_chunk_000000_indptr", "nnz": 95, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000073_chunk_000001_indices", "indptr": "tensor_000073_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000073_chunk_000002_indices", "indptr": "tensor_000073_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000073_chunk_000003_indices", "indptr": "tensor_000073_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 95, "shape": [ 2048, 2048 ] }, "model.layers.18.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000072_chunk_000000_indices", "indptr": "tensor_000072_chunk_000000_indptr", "nnz": 256, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000072_chunk_000001_indices", "indptr": "tensor_000072_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000072_chunk_000002_indices", "indptr": "tensor_000072_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000072_chunk_000003_indices", "indptr": "tensor_000072_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000072_chunk_000004_indices", "indptr": "tensor_000072_chunk_000004_indptr", "nnz": 256, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000072_chunk_000005_indices", "indptr": "tensor_000072_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000072_chunk_000006_indices", "indptr": "tensor_000072_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000072_chunk_000007_indices", "indptr": "tensor_000072_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000072_chunk_000008_indices", "indptr": "tensor_000072_chunk_000008_indptr", "nnz": 95, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000072_chunk_000009_indices", "indptr": "tensor_000072_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000072_chunk_000010_indices", "indptr": "tensor_000072_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000072_chunk_000011_indices", "indptr": "tensor_000072_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 607, "shape": [ 6144, 2048 ] }, "model.layers.19.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000156_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.19.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000079_chunk_000000_indices", "indptr": "tensor_000079_chunk_000000_indptr", "nnz": 412, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000079_chunk_000001_indices", "indptr": "tensor_000079_chunk_000001_indptr", "nnz": 195, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000079_chunk_000002_indices", "indptr": "tensor_000079_chunk_000002_indptr", "nnz": 0, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000079_chunk_000003_indices", "indptr": "tensor_000079_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000079_chunk_000004_indices", "indptr": "tensor_000079_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000079_chunk_000005_indices", "indptr": "tensor_000079_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000079_chunk_000006_indices", "indptr": "tensor_000079_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 607, "shape": [ 2048, 3415 ] }, "model.layers.19.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000078_chunk_000000_indices", "indptr": "tensor_000078_chunk_000000_indptr", "nnz": 1046, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000078_chunk_000001_indices", "indptr": "tensor_000078_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000078_chunk_000002_indices", "indptr": "tensor_000078_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000078_chunk_000003_indices", "indptr": "tensor_000078_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000078_chunk_000004_indices", "indptr": "tensor_000078_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000078_chunk_000005_indices", "indptr": "tensor_000078_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000078_chunk_000006_indices", "indptr": "tensor_000078_chunk_000006_indptr", "nnz": 169, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000078_chunk_000007_indices", "indptr": "tensor_000078_chunk_000007_indptr", "nnz": 277, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000078_chunk_000008_indices", "indptr": "tensor_000078_chunk_000008_indptr", "nnz": 0, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000078_chunk_000009_indices", "indptr": "tensor_000078_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000078_chunk_000010_indices", "indptr": "tensor_000078_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000078_chunk_000011_indices", "indptr": "tensor_000078_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000078_chunk_000012_indices", "indptr": "tensor_000078_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000078_chunk_000013_indices", "indptr": "tensor_000078_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 1492, "shape": [ 6830, 2048 ] }, "model.layers.19.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000157_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.19.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000077_chunk_000000_indices", "indptr": "tensor_000077_chunk_000000_indptr", "nnz": 75, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000077_chunk_000001_indices", "indptr": "tensor_000077_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000077_chunk_000002_indices", "indptr": "tensor_000077_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000077_chunk_000003_indices", "indptr": "tensor_000077_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 75, "shape": [ 2048, 2048 ] }, "model.layers.19.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000076_chunk_000000_indices", "indptr": "tensor_000076_chunk_000000_indptr", "nnz": 192, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000076_chunk_000001_indices", "indptr": "tensor_000076_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000076_chunk_000002_indices", "indptr": "tensor_000076_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000076_chunk_000003_indices", "indptr": "tensor_000076_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000076_chunk_000004_indices", "indptr": "tensor_000076_chunk_000004_indptr", "nnz": 192, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000076_chunk_000005_indices", "indptr": "tensor_000076_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000076_chunk_000006_indices", "indptr": "tensor_000076_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000076_chunk_000007_indices", "indptr": "tensor_000076_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000076_chunk_000008_indices", "indptr": "tensor_000076_chunk_000008_indptr", "nnz": 75, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000076_chunk_000009_indices", "indptr": "tensor_000076_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000076_chunk_000010_indices", "indptr": "tensor_000076_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000076_chunk_000011_indices", "indptr": "tensor_000076_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 459, "shape": [ 6144, 2048 ] }, "model.layers.2.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000122_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.2.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000011_chunk_000000_indices", "indptr": "tensor_000011_chunk_000000_indptr", "nnz": 75, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000011_chunk_000001_indices", "indptr": "tensor_000011_chunk_000001_indptr", "nnz": 0, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000011_chunk_000002_indices", "indptr": "tensor_000011_chunk_000002_indptr", "nnz": 0, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000011_chunk_000003_indices", "indptr": "tensor_000011_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000011_chunk_000004_indices", "indptr": "tensor_000011_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000011_chunk_000005_indices", "indptr": "tensor_000011_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000011_chunk_000006_indices", "indptr": "tensor_000011_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 75, "shape": [ 2048, 3415 ] }, "model.layers.2.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000010_chunk_000000_indices", "indptr": "tensor_000010_chunk_000000_indptr", "nnz": 143, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000010_chunk_000001_indices", "indptr": "tensor_000010_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000010_chunk_000002_indices", "indptr": "tensor_000010_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000010_chunk_000003_indices", "indptr": "tensor_000010_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000010_chunk_000004_indices", "indptr": "tensor_000010_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000010_chunk_000005_indices", "indptr": "tensor_000010_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000010_chunk_000006_indices", "indptr": "tensor_000010_chunk_000006_indptr", "nnz": 73, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000010_chunk_000007_indices", "indptr": "tensor_000010_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000010_chunk_000008_indices", "indptr": "tensor_000010_chunk_000008_indptr", "nnz": 0, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000010_chunk_000009_indices", "indptr": "tensor_000010_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000010_chunk_000010_indices", "indptr": "tensor_000010_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000010_chunk_000011_indices", "indptr": "tensor_000010_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000010_chunk_000012_indices", "indptr": "tensor_000010_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000010_chunk_000013_indices", "indptr": "tensor_000010_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 216, "shape": [ 6830, 2048 ] }, "model.layers.2.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000123_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.2.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000009_chunk_000000_indices", "indptr": "tensor_000009_chunk_000000_indptr", "nnz": 8, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000009_chunk_000001_indices", "indptr": "tensor_000009_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000009_chunk_000002_indices", "indptr": "tensor_000009_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000009_chunk_000003_indices", "indptr": "tensor_000009_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 8, "shape": [ 2048, 2048 ] }, "model.layers.2.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000008_chunk_000000_indices", "indptr": "tensor_000008_chunk_000000_indptr", "nnz": 66, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000008_chunk_000001_indices", "indptr": "tensor_000008_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000008_chunk_000002_indices", "indptr": "tensor_000008_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000008_chunk_000003_indices", "indptr": "tensor_000008_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000008_chunk_000004_indices", "indptr": "tensor_000008_chunk_000004_indptr", "nnz": 66, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000008_chunk_000005_indices", "indptr": "tensor_000008_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000008_chunk_000006_indices", "indptr": "tensor_000008_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000008_chunk_000007_indices", "indptr": "tensor_000008_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000008_chunk_000008_indices", "indptr": "tensor_000008_chunk_000008_indptr", "nnz": 8, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000008_chunk_000009_indices", "indptr": "tensor_000008_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000008_chunk_000010_indices", "indptr": "tensor_000008_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000008_chunk_000011_indices", "indptr": "tensor_000008_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 140, "shape": [ 6144, 2048 ] }, "model.layers.20.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000158_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.20.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000083_chunk_000000_indices", "indptr": "tensor_000083_chunk_000000_indptr", "nnz": 125, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000083_chunk_000001_indices", "indptr": "tensor_000083_chunk_000001_indptr", "nnz": 5, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000083_chunk_000002_indices", "indptr": "tensor_000083_chunk_000002_indptr", "nnz": 0, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000083_chunk_000003_indices", "indptr": "tensor_000083_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000083_chunk_000004_indices", "indptr": "tensor_000083_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000083_chunk_000005_indices", "indptr": "tensor_000083_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000083_chunk_000006_indices", "indptr": "tensor_000083_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 130, "shape": [ 2048, 3415 ] }, "model.layers.20.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000082_chunk_000000_indices", "indptr": "tensor_000082_chunk_000000_indptr", "nnz": 128, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000082_chunk_000001_indices", "indptr": "tensor_000082_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000082_chunk_000002_indices", "indptr": "tensor_000082_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000082_chunk_000003_indices", "indptr": "tensor_000082_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000082_chunk_000004_indices", "indptr": "tensor_000082_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000082_chunk_000005_indices", "indptr": "tensor_000082_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000082_chunk_000006_indices", "indptr": "tensor_000082_chunk_000006_indptr", "nnz": 128, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000082_chunk_000007_indices", "indptr": "tensor_000082_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000082_chunk_000008_indices", "indptr": "tensor_000082_chunk_000008_indptr", "nnz": 0, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000082_chunk_000009_indices", "indptr": "tensor_000082_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000082_chunk_000010_indices", "indptr": "tensor_000082_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000082_chunk_000011_indices", "indptr": "tensor_000082_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000082_chunk_000012_indices", "indptr": "tensor_000082_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000082_chunk_000013_indices", "indptr": "tensor_000082_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 256, "shape": [ 6830, 2048 ] }, "model.layers.20.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000159_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.20.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000081_chunk_000000_indices", "indptr": "tensor_000081_chunk_000000_indptr", "nnz": 172, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000081_chunk_000001_indices", "indptr": "tensor_000081_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000081_chunk_000002_indices", "indptr": "tensor_000081_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000081_chunk_000003_indices", "indptr": "tensor_000081_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 172, "shape": [ 2048, 2048 ] }, "model.layers.20.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000080_chunk_000000_indices", "indptr": "tensor_000080_chunk_000000_indptr", "nnz": 256, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000080_chunk_000001_indices", "indptr": "tensor_000080_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000080_chunk_000002_indices", "indptr": "tensor_000080_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000080_chunk_000003_indices", "indptr": "tensor_000080_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000080_chunk_000004_indices", "indptr": "tensor_000080_chunk_000004_indptr", "nnz": 256, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000080_chunk_000005_indices", "indptr": "tensor_000080_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000080_chunk_000006_indices", "indptr": "tensor_000080_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000080_chunk_000007_indices", "indptr": "tensor_000080_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000080_chunk_000008_indices", "indptr": "tensor_000080_chunk_000008_indptr", "nnz": 172, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000080_chunk_000009_indices", "indptr": "tensor_000080_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000080_chunk_000010_indices", "indptr": "tensor_000080_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000080_chunk_000011_indices", "indptr": "tensor_000080_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 684, "shape": [ 6144, 2048 ] }, "model.layers.21.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000160_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.21.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000087_chunk_000000_indices", "indptr": "tensor_000087_chunk_000000_indptr", "nnz": 114, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000087_chunk_000001_indices", "indptr": "tensor_000087_chunk_000001_indptr", "nnz": 2, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000087_chunk_000002_indices", "indptr": "tensor_000087_chunk_000002_indptr", "nnz": 0, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000087_chunk_000003_indices", "indptr": "tensor_000087_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000087_chunk_000004_indices", "indptr": "tensor_000087_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000087_chunk_000005_indices", "indptr": "tensor_000087_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000087_chunk_000006_indices", "indptr": "tensor_000087_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 116, "shape": [ 2048, 3415 ] }, "model.layers.21.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000086_chunk_000000_indices", "indptr": "tensor_000086_chunk_000000_indptr", "nnz": 113, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000086_chunk_000001_indices", "indptr": "tensor_000086_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000086_chunk_000002_indices", "indptr": "tensor_000086_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000086_chunk_000003_indices", "indptr": "tensor_000086_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000086_chunk_000004_indices", "indptr": "tensor_000086_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000086_chunk_000005_indices", "indptr": "tensor_000086_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000086_chunk_000006_indices", "indptr": "tensor_000086_chunk_000006_indptr", "nnz": 113, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000086_chunk_000007_indices", "indptr": "tensor_000086_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000086_chunk_000008_indices", "indptr": "tensor_000086_chunk_000008_indptr", "nnz": 0, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000086_chunk_000009_indices", "indptr": "tensor_000086_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000086_chunk_000010_indices", "indptr": "tensor_000086_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000086_chunk_000011_indices", "indptr": "tensor_000086_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000086_chunk_000012_indices", "indptr": "tensor_000086_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000086_chunk_000013_indices", "indptr": "tensor_000086_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 226, "shape": [ 6830, 2048 ] }, "model.layers.21.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000161_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.21.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000085_chunk_000000_indices", "indptr": "tensor_000085_chunk_000000_indptr", "nnz": 51, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000085_chunk_000001_indices", "indptr": "tensor_000085_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000085_chunk_000002_indices", "indptr": "tensor_000085_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000085_chunk_000003_indices", "indptr": "tensor_000085_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 51, "shape": [ 2048, 2048 ] }, "model.layers.21.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000084_chunk_000000_indices", "indptr": "tensor_000084_chunk_000000_indptr", "nnz": 64, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000084_chunk_000001_indices", "indptr": "tensor_000084_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000084_chunk_000002_indices", "indptr": "tensor_000084_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000084_chunk_000003_indices", "indptr": "tensor_000084_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000084_chunk_000004_indices", "indptr": "tensor_000084_chunk_000004_indptr", "nnz": 64, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000084_chunk_000005_indices", "indptr": "tensor_000084_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000084_chunk_000006_indices", "indptr": "tensor_000084_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000084_chunk_000007_indices", "indptr": "tensor_000084_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000084_chunk_000008_indices", "indptr": "tensor_000084_chunk_000008_indptr", "nnz": 51, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000084_chunk_000009_indices", "indptr": "tensor_000084_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000084_chunk_000010_indices", "indptr": "tensor_000084_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000084_chunk_000011_indices", "indptr": "tensor_000084_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 179, "shape": [ 6144, 2048 ] }, "model.layers.22.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000162_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.22.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000091_chunk_000000_indices", "indptr": "tensor_000091_chunk_000000_indptr", "nnz": 27, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000091_chunk_000001_indices", "indptr": "tensor_000091_chunk_000001_indptr", "nnz": 0, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000091_chunk_000002_indices", "indptr": "tensor_000091_chunk_000002_indptr", "nnz": 0, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000091_chunk_000003_indices", "indptr": "tensor_000091_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000091_chunk_000004_indices", "indptr": "tensor_000091_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000091_chunk_000005_indices", "indptr": "tensor_000091_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000091_chunk_000006_indices", "indptr": "tensor_000091_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 27, "shape": [ 2048, 3415 ] }, "model.layers.22.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000090_chunk_000000_indices", "indptr": "tensor_000090_chunk_000000_indptr", "nnz": 55, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000090_chunk_000001_indices", "indptr": "tensor_000090_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000090_chunk_000002_indices", "indptr": "tensor_000090_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000090_chunk_000003_indices", "indptr": "tensor_000090_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000090_chunk_000004_indices", "indptr": "tensor_000090_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000090_chunk_000005_indices", "indptr": "tensor_000090_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000090_chunk_000006_indices", "indptr": "tensor_000090_chunk_000006_indptr", "nnz": 23, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000090_chunk_000007_indices", "indptr": "tensor_000090_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000090_chunk_000008_indices", "indptr": "tensor_000090_chunk_000008_indptr", "nnz": 0, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000090_chunk_000009_indices", "indptr": "tensor_000090_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000090_chunk_000010_indices", "indptr": "tensor_000090_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000090_chunk_000011_indices", "indptr": "tensor_000090_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000090_chunk_000012_indices", "indptr": "tensor_000090_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000090_chunk_000013_indices", "indptr": "tensor_000090_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 78, "shape": [ 6830, 2048 ] }, "model.layers.22.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000163_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.22.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000089_chunk_000000_indices", "indptr": "tensor_000089_chunk_000000_indptr", "nnz": 1, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000089_chunk_000001_indices", "indptr": "tensor_000089_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000089_chunk_000002_indices", "indptr": "tensor_000089_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000089_chunk_000003_indices", "indptr": "tensor_000089_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 1, "shape": [ 2048, 2048 ] }, "model.layers.22.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000088_chunk_000000_indices", "indptr": "tensor_000088_chunk_000000_indptr", "nnz": 64, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000088_chunk_000001_indices", "indptr": "tensor_000088_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000088_chunk_000002_indices", "indptr": "tensor_000088_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000088_chunk_000003_indices", "indptr": "tensor_000088_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000088_chunk_000004_indices", "indptr": "tensor_000088_chunk_000004_indptr", "nnz": 64, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000088_chunk_000005_indices", "indptr": "tensor_000088_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000088_chunk_000006_indices", "indptr": "tensor_000088_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000088_chunk_000007_indices", "indptr": "tensor_000088_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000088_chunk_000008_indices", "indptr": "tensor_000088_chunk_000008_indptr", "nnz": 1, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000088_chunk_000009_indices", "indptr": "tensor_000088_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000088_chunk_000010_indices", "indptr": "tensor_000088_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000088_chunk_000011_indices", "indptr": "tensor_000088_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 129, "shape": [ 6144, 2048 ] }, "model.layers.23.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000164_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.23.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000095_chunk_000000_indices", "indptr": "tensor_000095_chunk_000000_indptr", "nnz": 10, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000095_chunk_000001_indices", "indptr": "tensor_000095_chunk_000001_indptr", "nnz": 0, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000095_chunk_000002_indices", "indptr": "tensor_000095_chunk_000002_indptr", "nnz": 0, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000095_chunk_000003_indices", "indptr": "tensor_000095_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000095_chunk_000004_indices", "indptr": "tensor_000095_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000095_chunk_000005_indices", "indptr": "tensor_000095_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000095_chunk_000006_indices", "indptr": "tensor_000095_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 10, "shape": [ 2048, 3415 ] }, "model.layers.23.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000094_chunk_000000_indices", "indptr": "tensor_000094_chunk_000000_indptr", "nnz": 13, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000094_chunk_000001_indices", "indptr": "tensor_000094_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000094_chunk_000002_indices", "indptr": "tensor_000094_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000094_chunk_000003_indices", "indptr": "tensor_000094_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000094_chunk_000004_indices", "indptr": "tensor_000094_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000094_chunk_000005_indices", "indptr": "tensor_000094_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000094_chunk_000006_indices", "indptr": "tensor_000094_chunk_000006_indptr", "nnz": 9, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000094_chunk_000007_indices", "indptr": "tensor_000094_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000094_chunk_000008_indices", "indptr": "tensor_000094_chunk_000008_indptr", "nnz": 0, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000094_chunk_000009_indices", "indptr": "tensor_000094_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000094_chunk_000010_indices", "indptr": "tensor_000094_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000094_chunk_000011_indices", "indptr": "tensor_000094_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000094_chunk_000012_indices", "indptr": "tensor_000094_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000094_chunk_000013_indices", "indptr": "tensor_000094_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 22, "shape": [ 6830, 2048 ] }, "model.layers.23.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000165_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.23.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000093_chunk_000000_indices", "indptr": "tensor_000093_chunk_000000_indptr", "nnz": 1, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000093_chunk_000001_indices", "indptr": "tensor_000093_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000093_chunk_000002_indices", "indptr": "tensor_000093_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000093_chunk_000003_indices", "indptr": "tensor_000093_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 1, "shape": [ 2048, 2048 ] }, "model.layers.23.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000092_chunk_000000_indices", "indptr": "tensor_000092_chunk_000000_indptr", "nnz": 64, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000092_chunk_000001_indices", "indptr": "tensor_000092_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000092_chunk_000002_indices", "indptr": "tensor_000092_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000092_chunk_000003_indices", "indptr": "tensor_000092_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000092_chunk_000004_indices", "indptr": "tensor_000092_chunk_000004_indptr", "nnz": 64, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000092_chunk_000005_indices", "indptr": "tensor_000092_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000092_chunk_000006_indices", "indptr": "tensor_000092_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000092_chunk_000007_indices", "indptr": "tensor_000092_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000092_chunk_000008_indices", "indptr": "tensor_000092_chunk_000008_indptr", "nnz": 1, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000092_chunk_000009_indices", "indptr": "tensor_000092_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000092_chunk_000010_indices", "indptr": "tensor_000092_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000092_chunk_000011_indices", "indptr": "tensor_000092_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 129, "shape": [ 6144, 2048 ] }, "model.layers.24.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000166_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.24.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000099_chunk_000000_indices", "indptr": "tensor_000099_chunk_000000_indptr", "nnz": 5, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000099_chunk_000001_indices", "indptr": "tensor_000099_chunk_000001_indptr", "nnz": 0, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000099_chunk_000002_indices", "indptr": "tensor_000099_chunk_000002_indptr", "nnz": 0, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000099_chunk_000003_indices", "indptr": "tensor_000099_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000099_chunk_000004_indices", "indptr": "tensor_000099_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000099_chunk_000005_indices", "indptr": "tensor_000099_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000099_chunk_000006_indices", "indptr": "tensor_000099_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 5, "shape": [ 2048, 3415 ] }, "model.layers.24.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000098_chunk_000000_indices", "indptr": "tensor_000098_chunk_000000_indptr", "nnz": 5, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000098_chunk_000001_indices", "indptr": "tensor_000098_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000098_chunk_000002_indices", "indptr": "tensor_000098_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000098_chunk_000003_indices", "indptr": "tensor_000098_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000098_chunk_000004_indices", "indptr": "tensor_000098_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000098_chunk_000005_indices", "indptr": "tensor_000098_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000098_chunk_000006_indices", "indptr": "tensor_000098_chunk_000006_indptr", "nnz": 5, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000098_chunk_000007_indices", "indptr": "tensor_000098_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000098_chunk_000008_indices", "indptr": "tensor_000098_chunk_000008_indptr", "nnz": 0, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000098_chunk_000009_indices", "indptr": "tensor_000098_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000098_chunk_000010_indices", "indptr": "tensor_000098_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000098_chunk_000011_indices", "indptr": "tensor_000098_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000098_chunk_000012_indices", "indptr": "tensor_000098_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000098_chunk_000013_indices", "indptr": "tensor_000098_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 10, "shape": [ 6830, 2048 ] }, "model.layers.24.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000167_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.24.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000097_chunk_000000_indices", "indptr": "tensor_000097_chunk_000000_indptr", "nnz": 9, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000097_chunk_000001_indices", "indptr": "tensor_000097_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000097_chunk_000002_indices", "indptr": "tensor_000097_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000097_chunk_000003_indices", "indptr": "tensor_000097_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 9, "shape": [ 2048, 2048 ] }, "model.layers.24.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000096_chunk_000000_indices", "indptr": "tensor_000096_chunk_000000_indptr", "nnz": 128, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000096_chunk_000001_indices", "indptr": "tensor_000096_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000096_chunk_000002_indices", "indptr": "tensor_000096_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000096_chunk_000003_indices", "indptr": "tensor_000096_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000096_chunk_000004_indices", "indptr": "tensor_000096_chunk_000004_indptr", "nnz": 128, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000096_chunk_000005_indices", "indptr": "tensor_000096_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000096_chunk_000006_indices", "indptr": "tensor_000096_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000096_chunk_000007_indices", "indptr": "tensor_000096_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000096_chunk_000008_indices", "indptr": "tensor_000096_chunk_000008_indptr", "nnz": 9, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000096_chunk_000009_indices", "indptr": "tensor_000096_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000096_chunk_000010_indices", "indptr": "tensor_000096_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000096_chunk_000011_indices", "indptr": "tensor_000096_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 265, "shape": [ 6144, 2048 ] }, "model.layers.25.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000168_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.25.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000103_chunk_000000_indices", "indptr": "tensor_000103_chunk_000000_indptr", "nnz": 143, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000103_chunk_000001_indices", "indptr": "tensor_000103_chunk_000001_indptr", "nnz": 0, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000103_chunk_000002_indices", "indptr": "tensor_000103_chunk_000002_indptr", "nnz": 0, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000103_chunk_000003_indices", "indptr": "tensor_000103_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000103_chunk_000004_indices", "indptr": "tensor_000103_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000103_chunk_000005_indices", "indptr": "tensor_000103_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000103_chunk_000006_indices", "indptr": "tensor_000103_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 143, "shape": [ 2048, 3415 ] }, "model.layers.25.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000102_chunk_000000_indices", "indptr": "tensor_000102_chunk_000000_indptr", "nnz": 93, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000102_chunk_000001_indices", "indptr": "tensor_000102_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000102_chunk_000002_indices", "indptr": "tensor_000102_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000102_chunk_000003_indices", "indptr": "tensor_000102_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000102_chunk_000004_indices", "indptr": "tensor_000102_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000102_chunk_000005_indices", "indptr": "tensor_000102_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000102_chunk_000006_indices", "indptr": "tensor_000102_chunk_000006_indptr", "nnz": 47, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000102_chunk_000007_indices", "indptr": "tensor_000102_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000102_chunk_000008_indices", "indptr": "tensor_000102_chunk_000008_indptr", "nnz": 0, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000102_chunk_000009_indices", "indptr": "tensor_000102_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000102_chunk_000010_indices", "indptr": "tensor_000102_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000102_chunk_000011_indices", "indptr": "tensor_000102_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000102_chunk_000012_indices", "indptr": "tensor_000102_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000102_chunk_000013_indices", "indptr": "tensor_000102_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 140, "shape": [ 6830, 2048 ] }, "model.layers.25.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000169_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.25.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000101_chunk_000000_indices", "indptr": "tensor_000101_chunk_000000_indptr", "nnz": 1, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000101_chunk_000001_indices", "indptr": "tensor_000101_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000101_chunk_000002_indices", "indptr": "tensor_000101_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000101_chunk_000003_indices", "indptr": "tensor_000101_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 1, "shape": [ 2048, 2048 ] }, "model.layers.25.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000100_chunk_000000_indices", "indptr": "tensor_000100_chunk_000000_indptr", "nnz": 64, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000100_chunk_000001_indices", "indptr": "tensor_000100_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000100_chunk_000002_indices", "indptr": "tensor_000100_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000100_chunk_000003_indices", "indptr": "tensor_000100_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000100_chunk_000004_indices", "indptr": "tensor_000100_chunk_000004_indptr", "nnz": 64, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000100_chunk_000005_indices", "indptr": "tensor_000100_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000100_chunk_000006_indices", "indptr": "tensor_000100_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000100_chunk_000007_indices", "indptr": "tensor_000100_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000100_chunk_000008_indices", "indptr": "tensor_000100_chunk_000008_indptr", "nnz": 1, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000100_chunk_000009_indices", "indptr": "tensor_000100_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000100_chunk_000010_indices", "indptr": "tensor_000100_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000100_chunk_000011_indices", "indptr": "tensor_000100_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 129, "shape": [ 6144, 2048 ] }, "model.layers.26.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000170_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.26.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000107_chunk_000000_indices", "indptr": "tensor_000107_chunk_000000_indptr", "nnz": 421, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000107_chunk_000001_indices", "indptr": "tensor_000107_chunk_000001_indptr", "nnz": 266, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000107_chunk_000002_indices", "indptr": "tensor_000107_chunk_000002_indptr", "nnz": 287, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000107_chunk_000003_indices", "indptr": "tensor_000107_chunk_000003_indptr", "nnz": 307, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000107_chunk_000004_indices", "indptr": "tensor_000107_chunk_000004_indptr", "nnz": 307, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000107_chunk_000005_indices", "indptr": "tensor_000107_chunk_000005_indptr", "nnz": 307, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000107_chunk_000006_indices", "indptr": "tensor_000107_chunk_000006_indptr", "nnz": 67, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 1962, "shape": [ 2048, 3415 ] }, "model.layers.26.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000106_chunk_000000_indices", "indptr": "tensor_000106_chunk_000000_indptr", "nnz": 511, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000106_chunk_000001_indices", "indptr": "tensor_000106_chunk_000001_indptr", "nnz": 512, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000106_chunk_000002_indices", "indptr": "tensor_000106_chunk_000002_indptr", "nnz": 512, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000106_chunk_000003_indices", "indptr": "tensor_000106_chunk_000003_indptr", "nnz": 427, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000106_chunk_000004_indices", "indptr": "tensor_000106_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000106_chunk_000005_indices", "indptr": "tensor_000106_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000106_chunk_000006_indices", "indptr": "tensor_000106_chunk_000006_indptr", "nnz": 168, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000106_chunk_000007_indices", "indptr": "tensor_000106_chunk_000007_indptr", "nnz": 512, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000106_chunk_000008_indices", "indptr": "tensor_000106_chunk_000008_indptr", "nnz": 512, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000106_chunk_000009_indices", "indptr": "tensor_000106_chunk_000009_indptr", "nnz": 512, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000106_chunk_000010_indices", "indptr": "tensor_000106_chunk_000010_indptr", "nnz": 258, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000106_chunk_000011_indices", "indptr": "tensor_000106_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000106_chunk_000012_indices", "indptr": "tensor_000106_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000106_chunk_000013_indices", "indptr": "tensor_000106_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 3924, "shape": [ 6830, 2048 ] }, "model.layers.26.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000171_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.26.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000105_chunk_000000_indices", "indptr": "tensor_000105_chunk_000000_indptr", "nnz": 1, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000105_chunk_000001_indices", "indptr": "tensor_000105_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000105_chunk_000002_indices", "indptr": "tensor_000105_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000105_chunk_000003_indices", "indptr": "tensor_000105_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 1, "shape": [ 2048, 2048 ] }, "model.layers.26.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000104_chunk_000000_indices", "indptr": "tensor_000104_chunk_000000_indptr", "nnz": 64, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000104_chunk_000001_indices", "indptr": "tensor_000104_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000104_chunk_000002_indices", "indptr": "tensor_000104_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000104_chunk_000003_indices", "indptr": "tensor_000104_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000104_chunk_000004_indices", "indptr": "tensor_000104_chunk_000004_indptr", "nnz": 64, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000104_chunk_000005_indices", "indptr": "tensor_000104_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000104_chunk_000006_indices", "indptr": "tensor_000104_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000104_chunk_000007_indices", "indptr": "tensor_000104_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000104_chunk_000008_indices", "indptr": "tensor_000104_chunk_000008_indptr", "nnz": 1, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000104_chunk_000009_indices", "indptr": "tensor_000104_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000104_chunk_000010_indices", "indptr": "tensor_000104_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000104_chunk_000011_indices", "indptr": "tensor_000104_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 129, "shape": [ 6144, 2048 ] }, "model.layers.27.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000172_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.27.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000111_chunk_000000_indices", "indptr": "tensor_000111_chunk_000000_indptr", "nnz": 259, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000111_chunk_000001_indices", "indptr": "tensor_000111_chunk_000001_indptr", "nnz": 498, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000111_chunk_000002_indices", "indptr": "tensor_000111_chunk_000002_indptr", "nnz": 574, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000111_chunk_000003_indices", "indptr": "tensor_000111_chunk_000003_indptr", "nnz": 614, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000111_chunk_000004_indices", "indptr": "tensor_000111_chunk_000004_indptr", "nnz": 614, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000111_chunk_000005_indices", "indptr": "tensor_000111_chunk_000005_indptr", "nnz": 614, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000111_chunk_000006_indices", "indptr": "tensor_000111_chunk_000006_indptr", "nnz": 242, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 3415, "shape": [ 2048, 3415 ] }, "model.layers.27.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000110_chunk_000000_indices", "indptr": "tensor_000110_chunk_000000_indptr", "nnz": 512, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000110_chunk_000001_indices", "indptr": "tensor_000110_chunk_000001_indptr", "nnz": 512, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000110_chunk_000002_indices", "indptr": "tensor_000110_chunk_000002_indptr", "nnz": 512, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000110_chunk_000003_indices", "indptr": "tensor_000110_chunk_000003_indptr", "nnz": 512, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000110_chunk_000004_indices", "indptr": "tensor_000110_chunk_000004_indptr", "nnz": 512, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000110_chunk_000005_indices", "indptr": "tensor_000110_chunk_000005_indptr", "nnz": 512, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000110_chunk_000006_indices", "indptr": "tensor_000110_chunk_000006_indptr", "nnz": 512, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000110_chunk_000007_indices", "indptr": "tensor_000110_chunk_000007_indptr", "nnz": 512, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000110_chunk_000008_indices", "indptr": "tensor_000110_chunk_000008_indptr", "nnz": 512, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000110_chunk_000009_indices", "indptr": "tensor_000110_chunk_000009_indptr", "nnz": 512, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000110_chunk_000010_indices", "indptr": "tensor_000110_chunk_000010_indptr", "nnz": 512, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000110_chunk_000011_indices", "indptr": "tensor_000110_chunk_000011_indptr", "nnz": 512, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000110_chunk_000012_indices", "indptr": "tensor_000110_chunk_000012_indptr", "nnz": 512, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000110_chunk_000013_indices", "indptr": "tensor_000110_chunk_000013_indptr", "nnz": 174, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 6830, "shape": [ 6830, 2048 ] }, "model.layers.27.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000173_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.27.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000109_chunk_000000_indices", "indptr": "tensor_000109_chunk_000000_indptr", "nnz": 1632, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000109_chunk_000001_indices", "indptr": "tensor_000109_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000109_chunk_000002_indices", "indptr": "tensor_000109_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000109_chunk_000003_indices", "indptr": "tensor_000109_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 1632, "shape": [ 2048, 2048 ] }, "model.layers.27.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000108_chunk_000000_indices", "indptr": "tensor_000108_chunk_000000_indptr", "nnz": 512, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000108_chunk_000001_indices", "indptr": "tensor_000108_chunk_000001_indptr", "nnz": 512, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000108_chunk_000002_indices", "indptr": "tensor_000108_chunk_000002_indptr", "nnz": 512, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000108_chunk_000003_indices", "indptr": "tensor_000108_chunk_000003_indptr", "nnz": 512, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000108_chunk_000004_indices", "indptr": "tensor_000108_chunk_000004_indptr", "nnz": 512, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000108_chunk_000005_indices", "indptr": "tensor_000108_chunk_000005_indptr", "nnz": 512, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000108_chunk_000006_indices", "indptr": "tensor_000108_chunk_000006_indptr", "nnz": 512, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000108_chunk_000007_indices", "indptr": "tensor_000108_chunk_000007_indptr", "nnz": 512, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000108_chunk_000008_indices", "indptr": "tensor_000108_chunk_000008_indptr", "nnz": 408, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000108_chunk_000009_indices", "indptr": "tensor_000108_chunk_000009_indptr", "nnz": 408, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000108_chunk_000010_indices", "indptr": "tensor_000108_chunk_000010_indptr", "nnz": 408, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000108_chunk_000011_indices", "indptr": "tensor_000108_chunk_000011_indptr", "nnz": 408, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 5728, "shape": [ 6144, 2048 ] }, "model.layers.28.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000174_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.28.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000115_chunk_000000_indices", "indptr": "tensor_000115_chunk_000000_indptr", "nnz": 123, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000115_chunk_000001_indices", "indptr": "tensor_000115_chunk_000001_indptr", "nnz": 4, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000115_chunk_000002_indices", "indptr": "tensor_000115_chunk_000002_indptr", "nnz": 0, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000115_chunk_000003_indices", "indptr": "tensor_000115_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000115_chunk_000004_indices", "indptr": "tensor_000115_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000115_chunk_000005_indices", "indptr": "tensor_000115_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000115_chunk_000006_indices", "indptr": "tensor_000115_chunk_000006_indptr", "nnz": 180, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 307, "shape": [ 2048, 3415 ] }, "model.layers.28.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000114_chunk_000000_indices", "indptr": "tensor_000114_chunk_000000_indptr", "nnz": 307, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000114_chunk_000001_indices", "indptr": "tensor_000114_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000114_chunk_000002_indices", "indptr": "tensor_000114_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000114_chunk_000003_indices", "indptr": "tensor_000114_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000114_chunk_000004_indices", "indptr": "tensor_000114_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000114_chunk_000005_indices", "indptr": "tensor_000114_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000114_chunk_000006_indices", "indptr": "tensor_000114_chunk_000006_indptr", "nnz": 169, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000114_chunk_000007_indices", "indptr": "tensor_000114_chunk_000007_indptr", "nnz": 138, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000114_chunk_000008_indices", "indptr": "tensor_000114_chunk_000008_indptr", "nnz": 0, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000114_chunk_000009_indices", "indptr": "tensor_000114_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000114_chunk_000010_indices", "indptr": "tensor_000114_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000114_chunk_000011_indices", "indptr": "tensor_000114_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000114_chunk_000012_indices", "indptr": "tensor_000114_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000114_chunk_000013_indices", "indptr": "tensor_000114_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 614, "shape": [ 6830, 2048 ] }, "model.layers.28.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000175_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.28.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000113_chunk_000000_indices", "indptr": "tensor_000113_chunk_000000_indptr", "nnz": 232, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000113_chunk_000001_indices", "indptr": "tensor_000113_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000113_chunk_000002_indices", "indptr": "tensor_000113_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000113_chunk_000003_indices", "indptr": "tensor_000113_chunk_000003_indptr", "nnz": 41, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 273, "shape": [ 2048, 2048 ] }, "model.layers.28.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000112_chunk_000000_indices", "indptr": "tensor_000112_chunk_000000_indptr", "nnz": 384, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000112_chunk_000001_indices", "indptr": "tensor_000112_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000112_chunk_000002_indices", "indptr": "tensor_000112_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000112_chunk_000003_indices", "indptr": "tensor_000112_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000112_chunk_000004_indices", "indptr": "tensor_000112_chunk_000004_indptr", "nnz": 384, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000112_chunk_000005_indices", "indptr": "tensor_000112_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000112_chunk_000006_indices", "indptr": "tensor_000112_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000112_chunk_000007_indices", "indptr": "tensor_000112_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000112_chunk_000008_indices", "indptr": "tensor_000112_chunk_000008_indptr", "nnz": 273, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000112_chunk_000009_indices", "indptr": "tensor_000112_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000112_chunk_000010_indices", "indptr": "tensor_000112_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000112_chunk_000011_indices", "indptr": "tensor_000112_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 1041, "shape": [ 6144, 2048 ] }, "model.layers.3.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000124_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.3.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000015_chunk_000000_indices", "indptr": "tensor_000015_chunk_000000_indptr", "nnz": 57, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000015_chunk_000001_indices", "indptr": "tensor_000015_chunk_000001_indptr", "nnz": 0, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000015_chunk_000002_indices", "indptr": "tensor_000015_chunk_000002_indptr", "nnz": 0, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000015_chunk_000003_indices", "indptr": "tensor_000015_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000015_chunk_000004_indices", "indptr": "tensor_000015_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000015_chunk_000005_indices", "indptr": "tensor_000015_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000015_chunk_000006_indices", "indptr": "tensor_000015_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 57, "shape": [ 2048, 3415 ] }, "model.layers.3.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000014_chunk_000000_indices", "indptr": "tensor_000014_chunk_000000_indptr", "nnz": 43, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000014_chunk_000001_indices", "indptr": "tensor_000014_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000014_chunk_000002_indices", "indptr": "tensor_000014_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000014_chunk_000003_indices", "indptr": "tensor_000014_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000014_chunk_000004_indices", "indptr": "tensor_000014_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000014_chunk_000005_indices", "indptr": "tensor_000014_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000014_chunk_000006_indices", "indptr": "tensor_000014_chunk_000006_indptr", "nnz": 25, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000014_chunk_000007_indices", "indptr": "tensor_000014_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000014_chunk_000008_indices", "indptr": "tensor_000014_chunk_000008_indptr", "nnz": 0, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000014_chunk_000009_indices", "indptr": "tensor_000014_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000014_chunk_000010_indices", "indptr": "tensor_000014_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000014_chunk_000011_indices", "indptr": "tensor_000014_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000014_chunk_000012_indices", "indptr": "tensor_000014_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000014_chunk_000013_indices", "indptr": "tensor_000014_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 68, "shape": [ 6830, 2048 ] }, "model.layers.3.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000125_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.3.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000013_chunk_000000_indices", "indptr": "tensor_000013_chunk_000000_indptr", "nnz": 4, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000013_chunk_000001_indices", "indptr": "tensor_000013_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000013_chunk_000002_indices", "indptr": "tensor_000013_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000013_chunk_000003_indices", "indptr": "tensor_000013_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 4, "shape": [ 2048, 2048 ] }, "model.layers.3.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000012_chunk_000000_indices", "indptr": "tensor_000012_chunk_000000_indptr", "nnz": 192, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000012_chunk_000001_indices", "indptr": "tensor_000012_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000012_chunk_000002_indices", "indptr": "tensor_000012_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000012_chunk_000003_indices", "indptr": "tensor_000012_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000012_chunk_000004_indices", "indptr": "tensor_000012_chunk_000004_indptr", "nnz": 192, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000012_chunk_000005_indices", "indptr": "tensor_000012_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000012_chunk_000006_indices", "indptr": "tensor_000012_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000012_chunk_000007_indices", "indptr": "tensor_000012_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000012_chunk_000008_indices", "indptr": "tensor_000012_chunk_000008_indptr", "nnz": 4, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000012_chunk_000009_indices", "indptr": "tensor_000012_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000012_chunk_000010_indices", "indptr": "tensor_000012_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000012_chunk_000011_indices", "indptr": "tensor_000012_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 388, "shape": [ 6144, 2048 ] }, "model.layers.4.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000126_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.4.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000019_chunk_000000_indices", "indptr": "tensor_000019_chunk_000000_indptr", "nnz": 475, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000019_chunk_000001_indices", "indptr": "tensor_000019_chunk_000001_indptr", "nnz": 0, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000019_chunk_000002_indices", "indptr": "tensor_000019_chunk_000002_indptr", "nnz": 0, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000019_chunk_000003_indices", "indptr": "tensor_000019_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000019_chunk_000004_indices", "indptr": "tensor_000019_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000019_chunk_000005_indices", "indptr": "tensor_000019_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000019_chunk_000006_indices", "indptr": "tensor_000019_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 475, "shape": [ 2048, 3415 ] }, "model.layers.4.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000018_chunk_000000_indices", "indptr": "tensor_000018_chunk_000000_indptr", "nnz": 466, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000018_chunk_000001_indices", "indptr": "tensor_000018_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000018_chunk_000002_indices", "indptr": "tensor_000018_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000018_chunk_000003_indices", "indptr": "tensor_000018_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000018_chunk_000004_indices", "indptr": "tensor_000018_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000018_chunk_000005_indices", "indptr": "tensor_000018_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000018_chunk_000006_indices", "indptr": "tensor_000018_chunk_000006_indptr", "nnz": 169, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000018_chunk_000007_indices", "indptr": "tensor_000018_chunk_000007_indptr", "nnz": 207, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000018_chunk_000008_indices", "indptr": "tensor_000018_chunk_000008_indptr", "nnz": 0, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000018_chunk_000009_indices", "indptr": "tensor_000018_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000018_chunk_000010_indices", "indptr": "tensor_000018_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000018_chunk_000011_indices", "indptr": "tensor_000018_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000018_chunk_000012_indices", "indptr": "tensor_000018_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000018_chunk_000013_indices", "indptr": "tensor_000018_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 842, "shape": [ 6830, 2048 ] }, "model.layers.4.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000127_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.4.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000017_chunk_000000_indices", "indptr": "tensor_000017_chunk_000000_indptr", "nnz": 29, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000017_chunk_000001_indices", "indptr": "tensor_000017_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000017_chunk_000002_indices", "indptr": "tensor_000017_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000017_chunk_000003_indices", "indptr": "tensor_000017_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 29, "shape": [ 2048, 2048 ] }, "model.layers.4.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000016_chunk_000000_indices", "indptr": "tensor_000016_chunk_000000_indptr", "nnz": 194, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000016_chunk_000001_indices", "indptr": "tensor_000016_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000016_chunk_000002_indices", "indptr": "tensor_000016_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000016_chunk_000003_indices", "indptr": "tensor_000016_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000016_chunk_000004_indices", "indptr": "tensor_000016_chunk_000004_indptr", "nnz": 194, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000016_chunk_000005_indices", "indptr": "tensor_000016_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000016_chunk_000006_indices", "indptr": "tensor_000016_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000016_chunk_000007_indices", "indptr": "tensor_000016_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000016_chunk_000008_indices", "indptr": "tensor_000016_chunk_000008_indptr", "nnz": 29, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000016_chunk_000009_indices", "indptr": "tensor_000016_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000016_chunk_000010_indices", "indptr": "tensor_000016_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000016_chunk_000011_indices", "indptr": "tensor_000016_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 417, "shape": [ 6144, 2048 ] }, "model.layers.5.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000128_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.5.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000023_chunk_000000_indices", "indptr": "tensor_000023_chunk_000000_indptr", "nnz": 342, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000023_chunk_000001_indices", "indptr": "tensor_000023_chunk_000001_indptr", "nnz": 32, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000023_chunk_000002_indices", "indptr": "tensor_000023_chunk_000002_indptr", "nnz": 0, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000023_chunk_000003_indices", "indptr": "tensor_000023_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000023_chunk_000004_indices", "indptr": "tensor_000023_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000023_chunk_000005_indices", "indptr": "tensor_000023_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000023_chunk_000006_indices", "indptr": "tensor_000023_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 374, "shape": [ 2048, 3415 ] }, "model.layers.5.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000022_chunk_000000_indices", "indptr": "tensor_000022_chunk_000000_indptr", "nnz": 420, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000022_chunk_000001_indices", "indptr": "tensor_000022_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000022_chunk_000002_indices", "indptr": "tensor_000022_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000022_chunk_000003_indices", "indptr": "tensor_000022_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000022_chunk_000004_indices", "indptr": "tensor_000022_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000022_chunk_000005_indices", "indptr": "tensor_000022_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000022_chunk_000006_indices", "indptr": "tensor_000022_chunk_000006_indptr", "nnz": 169, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000022_chunk_000007_indices", "indptr": "tensor_000022_chunk_000007_indptr", "nnz": 195, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000022_chunk_000008_indices", "indptr": "tensor_000022_chunk_000008_indptr", "nnz": 0, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000022_chunk_000009_indices", "indptr": "tensor_000022_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000022_chunk_000010_indices", "indptr": "tensor_000022_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000022_chunk_000011_indices", "indptr": "tensor_000022_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000022_chunk_000012_indices", "indptr": "tensor_000022_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000022_chunk_000013_indices", "indptr": "tensor_000022_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 784, "shape": [ 6830, 2048 ] }, "model.layers.5.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000129_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.5.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000021_chunk_000000_indices", "indptr": "tensor_000021_chunk_000000_indptr", "nnz": 149, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000021_chunk_000001_indices", "indptr": "tensor_000021_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000021_chunk_000002_indices", "indptr": "tensor_000021_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000021_chunk_000003_indices", "indptr": "tensor_000021_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 149, "shape": [ 2048, 2048 ] }, "model.layers.5.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000020_chunk_000000_indices", "indptr": "tensor_000020_chunk_000000_indptr", "nnz": 193, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000020_chunk_000001_indices", "indptr": "tensor_000020_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000020_chunk_000002_indices", "indptr": "tensor_000020_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000020_chunk_000003_indices", "indptr": "tensor_000020_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000020_chunk_000004_indices", "indptr": "tensor_000020_chunk_000004_indptr", "nnz": 193, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000020_chunk_000005_indices", "indptr": "tensor_000020_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000020_chunk_000006_indices", "indptr": "tensor_000020_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000020_chunk_000007_indices", "indptr": "tensor_000020_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000020_chunk_000008_indices", "indptr": "tensor_000020_chunk_000008_indptr", "nnz": 149, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000020_chunk_000009_indices", "indptr": "tensor_000020_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000020_chunk_000010_indices", "indptr": "tensor_000020_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000020_chunk_000011_indices", "indptr": "tensor_000020_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 535, "shape": [ 6144, 2048 ] }, "model.layers.6.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000130_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.6.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000027_chunk_000000_indices", "indptr": "tensor_000027_chunk_000000_indptr", "nnz": 70, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000027_chunk_000001_indices", "indptr": "tensor_000027_chunk_000001_indptr", "nnz": 16, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000027_chunk_000002_indices", "indptr": "tensor_000027_chunk_000002_indptr", "nnz": 0, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000027_chunk_000003_indices", "indptr": "tensor_000027_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000027_chunk_000004_indices", "indptr": "tensor_000027_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000027_chunk_000005_indices", "indptr": "tensor_000027_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000027_chunk_000006_indices", "indptr": "tensor_000027_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 86, "shape": [ 2048, 3415 ] }, "model.layers.6.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000026_chunk_000000_indices", "indptr": "tensor_000026_chunk_000000_indptr", "nnz": 84, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000026_chunk_000001_indices", "indptr": "tensor_000026_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000026_chunk_000002_indices", "indptr": "tensor_000026_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000026_chunk_000003_indices", "indptr": "tensor_000026_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000026_chunk_000004_indices", "indptr": "tensor_000026_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000026_chunk_000005_indices", "indptr": "tensor_000026_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000026_chunk_000006_indices", "indptr": "tensor_000026_chunk_000006_indptr", "nnz": 82, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000026_chunk_000007_indices", "indptr": "tensor_000026_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000026_chunk_000008_indices", "indptr": "tensor_000026_chunk_000008_indptr", "nnz": 0, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000026_chunk_000009_indices", "indptr": "tensor_000026_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000026_chunk_000010_indices", "indptr": "tensor_000026_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000026_chunk_000011_indices", "indptr": "tensor_000026_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000026_chunk_000012_indices", "indptr": "tensor_000026_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000026_chunk_000013_indices", "indptr": "tensor_000026_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 166, "shape": [ 6830, 2048 ] }, "model.layers.6.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000131_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.6.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000025_chunk_000000_indices", "indptr": "tensor_000025_chunk_000000_indptr", "nnz": 6, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000025_chunk_000001_indices", "indptr": "tensor_000025_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000025_chunk_000002_indices", "indptr": "tensor_000025_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000025_chunk_000003_indices", "indptr": "tensor_000025_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 6, "shape": [ 2048, 2048 ] }, "model.layers.6.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000024_chunk_000000_indices", "indptr": "tensor_000024_chunk_000000_indptr", "nnz": 128, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000024_chunk_000001_indices", "indptr": "tensor_000024_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000024_chunk_000002_indices", "indptr": "tensor_000024_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000024_chunk_000003_indices", "indptr": "tensor_000024_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000024_chunk_000004_indices", "indptr": "tensor_000024_chunk_000004_indptr", "nnz": 128, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000024_chunk_000005_indices", "indptr": "tensor_000024_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000024_chunk_000006_indices", "indptr": "tensor_000024_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000024_chunk_000007_indices", "indptr": "tensor_000024_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000024_chunk_000008_indices", "indptr": "tensor_000024_chunk_000008_indptr", "nnz": 6, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000024_chunk_000009_indices", "indptr": "tensor_000024_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000024_chunk_000010_indices", "indptr": "tensor_000024_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000024_chunk_000011_indices", "indptr": "tensor_000024_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 262, "shape": [ 6144, 2048 ] }, "model.layers.7.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000132_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.7.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000031_chunk_000000_indices", "indptr": "tensor_000031_chunk_000000_indptr", "nnz": 87, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000031_chunk_000001_indices", "indptr": "tensor_000031_chunk_000001_indptr", "nnz": 48, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000031_chunk_000002_indices", "indptr": "tensor_000031_chunk_000002_indptr", "nnz": 0, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000031_chunk_000003_indices", "indptr": "tensor_000031_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000031_chunk_000004_indices", "indptr": "tensor_000031_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000031_chunk_000005_indices", "indptr": "tensor_000031_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000031_chunk_000006_indices", "indptr": "tensor_000031_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 135, "shape": [ 2048, 3415 ] }, "model.layers.7.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000030_chunk_000000_indices", "indptr": "tensor_000030_chunk_000000_indptr", "nnz": 105, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000030_chunk_000001_indices", "indptr": "tensor_000030_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000030_chunk_000002_indices", "indptr": "tensor_000030_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000030_chunk_000003_indices", "indptr": "tensor_000030_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000030_chunk_000004_indices", "indptr": "tensor_000030_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000030_chunk_000005_indices", "indptr": "tensor_000030_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000030_chunk_000006_indices", "indptr": "tensor_000030_chunk_000006_indptr", "nnz": 54, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000030_chunk_000007_indices", "indptr": "tensor_000030_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000030_chunk_000008_indices", "indptr": "tensor_000030_chunk_000008_indptr", "nnz": 0, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000030_chunk_000009_indices", "indptr": "tensor_000030_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000030_chunk_000010_indices", "indptr": "tensor_000030_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000030_chunk_000011_indices", "indptr": "tensor_000030_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000030_chunk_000012_indices", "indptr": "tensor_000030_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000030_chunk_000013_indices", "indptr": "tensor_000030_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 159, "shape": [ 6830, 2048 ] }, "model.layers.7.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000133_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.7.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000029_chunk_000000_indices", "indptr": "tensor_000029_chunk_000000_indptr", "nnz": 27, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000029_chunk_000001_indices", "indptr": "tensor_000029_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000029_chunk_000002_indices", "indptr": "tensor_000029_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000029_chunk_000003_indices", "indptr": "tensor_000029_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 27, "shape": [ 2048, 2048 ] }, "model.layers.7.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000028_chunk_000000_indices", "indptr": "tensor_000028_chunk_000000_indptr", "nnz": 203, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000028_chunk_000001_indices", "indptr": "tensor_000028_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000028_chunk_000002_indices", "indptr": "tensor_000028_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000028_chunk_000003_indices", "indptr": "tensor_000028_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000028_chunk_000004_indices", "indptr": "tensor_000028_chunk_000004_indptr", "nnz": 203, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000028_chunk_000005_indices", "indptr": "tensor_000028_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000028_chunk_000006_indices", "indptr": "tensor_000028_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000028_chunk_000007_indices", "indptr": "tensor_000028_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000028_chunk_000008_indices", "indptr": "tensor_000028_chunk_000008_indptr", "nnz": 27, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000028_chunk_000009_indices", "indptr": "tensor_000028_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000028_chunk_000010_indices", "indptr": "tensor_000028_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000028_chunk_000011_indices", "indptr": "tensor_000028_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 433, "shape": [ 6144, 2048 ] }, "model.layers.8.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000134_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.8.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000035_chunk_000000_indices", "indptr": "tensor_000035_chunk_000000_indptr", "nnz": 9, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000035_chunk_000001_indices", "indptr": "tensor_000035_chunk_000001_indptr", "nnz": 0, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000035_chunk_000002_indices", "indptr": "tensor_000035_chunk_000002_indptr", "nnz": 0, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000035_chunk_000003_indices", "indptr": "tensor_000035_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000035_chunk_000004_indices", "indptr": "tensor_000035_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000035_chunk_000005_indices", "indptr": "tensor_000035_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000035_chunk_000006_indices", "indptr": "tensor_000035_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 9, "shape": [ 2048, 3415 ] }, "model.layers.8.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000034_chunk_000000_indices", "indptr": "tensor_000034_chunk_000000_indptr", "nnz": 6, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000034_chunk_000001_indices", "indptr": "tensor_000034_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000034_chunk_000002_indices", "indptr": "tensor_000034_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000034_chunk_000003_indices", "indptr": "tensor_000034_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000034_chunk_000004_indices", "indptr": "tensor_000034_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000034_chunk_000005_indices", "indptr": "tensor_000034_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000034_chunk_000006_indices", "indptr": "tensor_000034_chunk_000006_indptr", "nnz": 9, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000034_chunk_000007_indices", "indptr": "tensor_000034_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000034_chunk_000008_indices", "indptr": "tensor_000034_chunk_000008_indptr", "nnz": 0, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000034_chunk_000009_indices", "indptr": "tensor_000034_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000034_chunk_000010_indices", "indptr": "tensor_000034_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000034_chunk_000011_indices", "indptr": "tensor_000034_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000034_chunk_000012_indices", "indptr": "tensor_000034_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000034_chunk_000013_indices", "indptr": "tensor_000034_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 15, "shape": [ 6830, 2048 ] }, "model.layers.8.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000135_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.8.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000033_chunk_000000_indices", "indptr": "tensor_000033_chunk_000000_indptr", "nnz": 96, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000033_chunk_000001_indices", "indptr": "tensor_000033_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000033_chunk_000002_indices", "indptr": "tensor_000033_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000033_chunk_000003_indices", "indptr": "tensor_000033_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 96, "shape": [ 2048, 2048 ] }, "model.layers.8.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000032_chunk_000000_indices", "indptr": "tensor_000032_chunk_000000_indptr", "nnz": 140, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000032_chunk_000001_indices", "indptr": "tensor_000032_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000032_chunk_000002_indices", "indptr": "tensor_000032_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000032_chunk_000003_indices", "indptr": "tensor_000032_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000032_chunk_000004_indices", "indptr": "tensor_000032_chunk_000004_indptr", "nnz": 140, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000032_chunk_000005_indices", "indptr": "tensor_000032_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000032_chunk_000006_indices", "indptr": "tensor_000032_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000032_chunk_000007_indices", "indptr": "tensor_000032_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000032_chunk_000008_indices", "indptr": "tensor_000032_chunk_000008_indptr", "nnz": 96, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000032_chunk_000009_indices", "indptr": "tensor_000032_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000032_chunk_000010_indices", "indptr": "tensor_000032_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000032_chunk_000011_indices", "indptr": "tensor_000032_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 376, "shape": [ 6144, 2048 ] }, "model.layers.9.input_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000136_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.9.mlp.down_proj.weight": { "chunks": [ { "indices": "tensor_000039_chunk_000000_indices", "indptr": "tensor_000039_chunk_000000_indptr", "nnz": 0, "row_count": 307, "row_start": 0 }, { "indices": "tensor_000039_chunk_000001_indices", "indptr": "tensor_000039_chunk_000001_indptr", "nnz": 0, "row_count": 307, "row_start": 307 }, { "indices": "tensor_000039_chunk_000002_indices", "indptr": "tensor_000039_chunk_000002_indptr", "nnz": 0, "row_count": 307, "row_start": 614 }, { "indices": "tensor_000039_chunk_000003_indices", "indptr": "tensor_000039_chunk_000003_indptr", "nnz": 0, "row_count": 307, "row_start": 921 }, { "indices": "tensor_000039_chunk_000004_indices", "indptr": "tensor_000039_chunk_000004_indptr", "nnz": 0, "row_count": 307, "row_start": 1228 }, { "indices": "tensor_000039_chunk_000005_indices", "indptr": "tensor_000039_chunk_000005_indptr", "nnz": 0, "row_count": 307, "row_start": 1535 }, { "indices": "tensor_000039_chunk_000006_indices", "indptr": "tensor_000039_chunk_000006_indptr", "nnz": 0, "row_count": 206, "row_start": 1842 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 0, "shape": [ 2048, 3415 ] }, "model.layers.9.mlp.gate_up_proj.weight": { "chunks": [ { "indices": "tensor_000038_chunk_000000_indices", "indptr": "tensor_000038_chunk_000000_indptr", "nnz": 0, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000038_chunk_000001_indices", "indptr": "tensor_000038_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000038_chunk_000002_indices", "indptr": "tensor_000038_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000038_chunk_000003_indices", "indptr": "tensor_000038_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000038_chunk_000004_indices", "indptr": "tensor_000038_chunk_000004_indptr", "nnz": 0, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000038_chunk_000005_indices", "indptr": "tensor_000038_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000038_chunk_000006_indices", "indptr": "tensor_000038_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000038_chunk_000007_indices", "indptr": "tensor_000038_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000038_chunk_000008_indices", "indptr": "tensor_000038_chunk_000008_indptr", "nnz": 0, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000038_chunk_000009_indices", "indptr": "tensor_000038_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000038_chunk_000010_indices", "indptr": "tensor_000038_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000038_chunk_000011_indices", "indptr": "tensor_000038_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 }, { "indices": "tensor_000038_chunk_000012_indices", "indptr": "tensor_000038_chunk_000012_indptr", "nnz": 0, "row_count": 512, "row_start": 6144 }, { "indices": "tensor_000038_chunk_000013_indices", "indptr": "tensor_000038_chunk_000013_indptr", "nnz": 0, "row_count": 174, "row_start": 6656 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 0, "shape": [ 6830, 2048 ] }, "model.layers.9.post_attention_layernorm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000137_indices", "nnz": 2048, "shape": [ 2048 ] }, "model.layers.9.self_attn.o_proj.weight": { "chunks": [ { "indices": "tensor_000037_chunk_000000_indices", "indptr": "tensor_000037_chunk_000000_indptr", "nnz": 7, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000037_chunk_000001_indices", "indptr": "tensor_000037_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000037_chunk_000002_indices", "indptr": "tensor_000037_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000037_chunk_000003_indices", "indptr": "tensor_000037_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 7, "shape": [ 2048, 2048 ] }, "model.layers.9.self_attn.qkv_proj.weight": { "chunks": [ { "indices": "tensor_000036_chunk_000000_indices", "indptr": "tensor_000036_chunk_000000_indptr", "nnz": 128, "row_count": 512, "row_start": 0 }, { "indices": "tensor_000036_chunk_000001_indices", "indptr": "tensor_000036_chunk_000001_indptr", "nnz": 0, "row_count": 512, "row_start": 512 }, { "indices": "tensor_000036_chunk_000002_indices", "indptr": "tensor_000036_chunk_000002_indptr", "nnz": 0, "row_count": 512, "row_start": 1024 }, { "indices": "tensor_000036_chunk_000003_indices", "indptr": "tensor_000036_chunk_000003_indptr", "nnz": 0, "row_count": 512, "row_start": 1536 }, { "indices": "tensor_000036_chunk_000004_indices", "indptr": "tensor_000036_chunk_000004_indptr", "nnz": 128, "row_count": 512, "row_start": 2048 }, { "indices": "tensor_000036_chunk_000005_indices", "indptr": "tensor_000036_chunk_000005_indptr", "nnz": 0, "row_count": 512, "row_start": 2560 }, { "indices": "tensor_000036_chunk_000006_indices", "indptr": "tensor_000036_chunk_000006_indptr", "nnz": 0, "row_count": 512, "row_start": 3072 }, { "indices": "tensor_000036_chunk_000007_indices", "indptr": "tensor_000036_chunk_000007_indptr", "nnz": 0, "row_count": 512, "row_start": 3584 }, { "indices": "tensor_000036_chunk_000008_indices", "indptr": "tensor_000036_chunk_000008_indptr", "nnz": 7, "row_count": 512, "row_start": 4096 }, { "indices": "tensor_000036_chunk_000009_indices", "indptr": "tensor_000036_chunk_000009_indptr", "nnz": 0, "row_count": 512, "row_start": 4608 }, { "indices": "tensor_000036_chunk_000010_indices", "indptr": "tensor_000036_chunk_000010_indptr", "nnz": 0, "row_count": 512, "row_start": 5120 }, { "indices": "tensor_000036_chunk_000011_indices", "indptr": "tensor_000036_chunk_000011_indptr", "nnz": 0, "row_count": 512, "row_start": 5632 } ], "dtype": "float32", "encoding": "csr_row_chunks", "nnz": 263, "shape": [ 6144, 2048 ] }, "model.norm.weight": { "dtype": "float32", "encoding": "flat_indices", "indices": "tensor_000117_indices", "nnz": 2046, "shape": [ 2048 ] } }, "zero_test": "stored fp32 value != 0.0" }, "physical_layout": { "checkpoint_parameter_map": [ { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 64 ], "checkpoint_tensor": "model.layers.0.self_attn.qkv_proj.weight", "logical_matrix": "L0.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 64 ], "checkpoint_tensor": "model.layers.0.self_attn.qkv_proj.weight", "logical_matrix": "L0.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 64 ], "checkpoint_tensor": "model.layers.0.self_attn.qkv_proj.weight", "logical_matrix": "L0.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 64 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.0.self_attn.o_proj.weight", "logical_matrix": "L0.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 16 ], "checkpoint_tensor": "model.layers.0.mlp.gate_up_proj.weight", "logical_matrix": "L0.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 16 ], "checkpoint_tensor": "model.layers.0.mlp.gate_up_proj.weight", "logical_matrix": "L0.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 16 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.0.mlp.down_proj.weight", "logical_matrix": "L0.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 256 ], "checkpoint_tensor": "model.layers.1.self_attn.qkv_proj.weight", "logical_matrix": "L1.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 256 ], "checkpoint_tensor": "model.layers.1.self_attn.qkv_proj.weight", "logical_matrix": "L1.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 256 ], "checkpoint_tensor": "model.layers.1.self_attn.qkv_proj.weight", "logical_matrix": "L1.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 256 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.1.self_attn.o_proj.weight", "logical_matrix": "L1.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 29 ], "checkpoint_tensor": "model.layers.1.mlp.gate_up_proj.weight", "logical_matrix": "L1.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 29 ], "checkpoint_tensor": "model.layers.1.mlp.gate_up_proj.weight", "logical_matrix": "L1.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 29 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.1.mlp.down_proj.weight", "logical_matrix": "L1.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 192 ], "checkpoint_tensor": "model.layers.2.self_attn.qkv_proj.weight", "logical_matrix": "L2.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 192 ], "checkpoint_tensor": "model.layers.2.self_attn.qkv_proj.weight", "logical_matrix": "L2.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 192 ], "checkpoint_tensor": "model.layers.2.self_attn.qkv_proj.weight", "logical_matrix": "L2.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 192 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.2.self_attn.o_proj.weight", "logical_matrix": "L2.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 73 ], "checkpoint_tensor": "model.layers.2.mlp.gate_up_proj.weight", "logical_matrix": "L2.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 73 ], "checkpoint_tensor": "model.layers.2.mlp.gate_up_proj.weight", "logical_matrix": "L2.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 73 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.2.mlp.down_proj.weight", "logical_matrix": "L2.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 192 ], "checkpoint_tensor": "model.layers.3.self_attn.qkv_proj.weight", "logical_matrix": "L3.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 192 ], "checkpoint_tensor": "model.layers.3.self_attn.qkv_proj.weight", "logical_matrix": "L3.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 192 ], "checkpoint_tensor": "model.layers.3.self_attn.qkv_proj.weight", "logical_matrix": "L3.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 192 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.3.self_attn.o_proj.weight", "logical_matrix": "L3.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 25 ], "checkpoint_tensor": "model.layers.3.mlp.gate_up_proj.weight", "logical_matrix": "L3.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 25 ], "checkpoint_tensor": "model.layers.3.mlp.gate_up_proj.weight", "logical_matrix": "L3.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 25 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.3.mlp.down_proj.weight", "logical_matrix": "L3.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 512 ], "checkpoint_tensor": "model.layers.4.self_attn.qkv_proj.weight", "logical_matrix": "L4.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 512 ], "checkpoint_tensor": "model.layers.4.self_attn.qkv_proj.weight", "logical_matrix": "L4.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 512 ], "checkpoint_tensor": "model.layers.4.self_attn.qkv_proj.weight", "logical_matrix": "L4.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 512 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.4.self_attn.o_proj.weight", "logical_matrix": "L4.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 376 ], "checkpoint_tensor": "model.layers.4.mlp.gate_up_proj.weight", "logical_matrix": "L4.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 376 ], "checkpoint_tensor": "model.layers.4.mlp.gate_up_proj.weight", "logical_matrix": "L4.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 376 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.4.mlp.down_proj.weight", "logical_matrix": "L4.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 256 ], "checkpoint_tensor": "model.layers.5.self_attn.qkv_proj.weight", "logical_matrix": "L5.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 256 ], "checkpoint_tensor": "model.layers.5.self_attn.qkv_proj.weight", "logical_matrix": "L5.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 256 ], "checkpoint_tensor": "model.layers.5.self_attn.qkv_proj.weight", "logical_matrix": "L5.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 256 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.5.self_attn.o_proj.weight", "logical_matrix": "L5.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 364 ], "checkpoint_tensor": "model.layers.5.mlp.gate_up_proj.weight", "logical_matrix": "L5.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 364 ], "checkpoint_tensor": "model.layers.5.mlp.gate_up_proj.weight", "logical_matrix": "L5.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 364 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.5.mlp.down_proj.weight", "logical_matrix": "L5.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 128 ], "checkpoint_tensor": "model.layers.6.self_attn.qkv_proj.weight", "logical_matrix": "L6.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 128 ], "checkpoint_tensor": "model.layers.6.self_attn.qkv_proj.weight", "logical_matrix": "L6.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 128 ], "checkpoint_tensor": "model.layers.6.self_attn.qkv_proj.weight", "logical_matrix": "L6.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 128 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.6.self_attn.o_proj.weight", "logical_matrix": "L6.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 82 ], "checkpoint_tensor": "model.layers.6.mlp.gate_up_proj.weight", "logical_matrix": "L6.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 82 ], "checkpoint_tensor": "model.layers.6.mlp.gate_up_proj.weight", "logical_matrix": "L6.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 82 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.6.mlp.down_proj.weight", "logical_matrix": "L6.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 256 ], "checkpoint_tensor": "model.layers.7.self_attn.qkv_proj.weight", "logical_matrix": "L7.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 256 ], "checkpoint_tensor": "model.layers.7.self_attn.qkv_proj.weight", "logical_matrix": "L7.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 256 ], "checkpoint_tensor": "model.layers.7.self_attn.qkv_proj.weight", "logical_matrix": "L7.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 256 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.7.self_attn.o_proj.weight", "logical_matrix": "L7.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 54 ], "checkpoint_tensor": "model.layers.7.mlp.gate_up_proj.weight", "logical_matrix": "L7.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 54 ], "checkpoint_tensor": "model.layers.7.mlp.gate_up_proj.weight", "logical_matrix": "L7.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 54 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.7.mlp.down_proj.weight", "logical_matrix": "L7.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 320 ], "checkpoint_tensor": "model.layers.8.self_attn.qkv_proj.weight", "logical_matrix": "L8.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 320 ], "checkpoint_tensor": "model.layers.8.self_attn.qkv_proj.weight", "logical_matrix": "L8.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 320 ], "checkpoint_tensor": "model.layers.8.self_attn.qkv_proj.weight", "logical_matrix": "L8.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 320 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.8.self_attn.o_proj.weight", "logical_matrix": "L8.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 6 ], "checkpoint_tensor": "model.layers.8.mlp.gate_up_proj.weight", "logical_matrix": "L8.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 6 ], "checkpoint_tensor": "model.layers.8.mlp.gate_up_proj.weight", "logical_matrix": "L8.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 6 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.8.mlp.down_proj.weight", "logical_matrix": "L8.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 128 ], "checkpoint_tensor": "model.layers.9.self_attn.qkv_proj.weight", "logical_matrix": "L9.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 128 ], "checkpoint_tensor": "model.layers.9.self_attn.qkv_proj.weight", "logical_matrix": "L9.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 128 ], "checkpoint_tensor": "model.layers.9.self_attn.qkv_proj.weight", "logical_matrix": "L9.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 128 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.9.self_attn.o_proj.weight", "logical_matrix": "L9.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 1 ], "checkpoint_tensor": "model.layers.9.mlp.gate_up_proj.weight", "logical_matrix": "L9.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 1 ], "checkpoint_tensor": "model.layers.9.mlp.gate_up_proj.weight", "logical_matrix": "L9.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 1 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.9.mlp.down_proj.weight", "logical_matrix": "L9.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 256 ], "checkpoint_tensor": "model.layers.10.self_attn.qkv_proj.weight", "logical_matrix": "L10.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 256 ], "checkpoint_tensor": "model.layers.10.self_attn.qkv_proj.weight", "logical_matrix": "L10.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 256 ], "checkpoint_tensor": "model.layers.10.self_attn.qkv_proj.weight", "logical_matrix": "L10.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 256 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.10.self_attn.o_proj.weight", "logical_matrix": "L10.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 3 ], "checkpoint_tensor": "model.layers.10.mlp.gate_up_proj.weight", "logical_matrix": "L10.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 3 ], "checkpoint_tensor": "model.layers.10.mlp.gate_up_proj.weight", "logical_matrix": "L10.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 3 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.10.mlp.down_proj.weight", "logical_matrix": "L10.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 512 ], "checkpoint_tensor": "model.layers.11.self_attn.qkv_proj.weight", "logical_matrix": "L11.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 512 ], "checkpoint_tensor": "model.layers.11.self_attn.qkv_proj.weight", "logical_matrix": "L11.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 512 ], "checkpoint_tensor": "model.layers.11.self_attn.qkv_proj.weight", "logical_matrix": "L11.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 512 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.11.self_attn.o_proj.weight", "logical_matrix": "L11.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 1543 ], "checkpoint_tensor": "model.layers.11.mlp.gate_up_proj.weight", "logical_matrix": "L11.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 1543 ], "checkpoint_tensor": "model.layers.11.mlp.gate_up_proj.weight", "logical_matrix": "L11.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 1543 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.11.mlp.down_proj.weight", "logical_matrix": "L11.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 768 ], "checkpoint_tensor": "model.layers.12.self_attn.qkv_proj.weight", "logical_matrix": "L12.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 768 ], "checkpoint_tensor": "model.layers.12.self_attn.qkv_proj.weight", "logical_matrix": "L12.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 768 ], "checkpoint_tensor": "model.layers.12.self_attn.qkv_proj.weight", "logical_matrix": "L12.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 768 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.12.self_attn.o_proj.weight", "logical_matrix": "L12.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 399 ], "checkpoint_tensor": "model.layers.12.mlp.gate_up_proj.weight", "logical_matrix": "L12.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 399 ], "checkpoint_tensor": "model.layers.12.mlp.gate_up_proj.weight", "logical_matrix": "L12.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 399 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.12.mlp.down_proj.weight", "logical_matrix": "L12.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 128 ], "checkpoint_tensor": "model.layers.13.self_attn.qkv_proj.weight", "logical_matrix": "L13.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 128 ], "checkpoint_tensor": "model.layers.13.self_attn.qkv_proj.weight", "logical_matrix": "L13.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 128 ], "checkpoint_tensor": "model.layers.13.self_attn.qkv_proj.weight", "logical_matrix": "L13.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 128 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.13.self_attn.o_proj.weight", "logical_matrix": "L13.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 1888 ], "checkpoint_tensor": "model.layers.13.mlp.gate_up_proj.weight", "logical_matrix": "L13.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 1888 ], "checkpoint_tensor": "model.layers.13.mlp.gate_up_proj.weight", "logical_matrix": "L13.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 1888 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.13.mlp.down_proj.weight", "logical_matrix": "L13.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 128 ], "checkpoint_tensor": "model.layers.14.self_attn.qkv_proj.weight", "logical_matrix": "L14.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 128 ], "checkpoint_tensor": "model.layers.14.self_attn.qkv_proj.weight", "logical_matrix": "L14.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 128 ], "checkpoint_tensor": "model.layers.14.self_attn.qkv_proj.weight", "logical_matrix": "L14.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 128 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.14.self_attn.o_proj.weight", "logical_matrix": "L14.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 2254 ], "checkpoint_tensor": "model.layers.14.mlp.gate_up_proj.weight", "logical_matrix": "L14.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 2254 ], "checkpoint_tensor": "model.layers.14.mlp.gate_up_proj.weight", "logical_matrix": "L14.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2254 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.14.mlp.down_proj.weight", "logical_matrix": "L14.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 384 ], "checkpoint_tensor": "model.layers.15.self_attn.qkv_proj.weight", "logical_matrix": "L15.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 384 ], "checkpoint_tensor": "model.layers.15.self_attn.qkv_proj.weight", "logical_matrix": "L15.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 384 ], "checkpoint_tensor": "model.layers.15.self_attn.qkv_proj.weight", "logical_matrix": "L15.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 384 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.15.self_attn.o_proj.weight", "logical_matrix": "L15.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 1274 ], "checkpoint_tensor": "model.layers.15.mlp.gate_up_proj.weight", "logical_matrix": "L15.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 1274 ], "checkpoint_tensor": "model.layers.15.mlp.gate_up_proj.weight", "logical_matrix": "L15.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 1274 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.15.mlp.down_proj.weight", "logical_matrix": "L15.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 256 ], "checkpoint_tensor": "model.layers.16.self_attn.qkv_proj.weight", "logical_matrix": "L16.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 256 ], "checkpoint_tensor": "model.layers.16.self_attn.qkv_proj.weight", "logical_matrix": "L16.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 256 ], "checkpoint_tensor": "model.layers.16.self_attn.qkv_proj.weight", "logical_matrix": "L16.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 256 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.16.self_attn.o_proj.weight", "logical_matrix": "L16.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 792 ], "checkpoint_tensor": "model.layers.16.mlp.gate_up_proj.weight", "logical_matrix": "L16.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 792 ], "checkpoint_tensor": "model.layers.16.mlp.gate_up_proj.weight", "logical_matrix": "L16.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 792 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.16.mlp.down_proj.weight", "logical_matrix": "L16.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 256 ], "checkpoint_tensor": "model.layers.17.self_attn.qkv_proj.weight", "logical_matrix": "L17.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 256 ], "checkpoint_tensor": "model.layers.17.self_attn.qkv_proj.weight", "logical_matrix": "L17.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 256 ], "checkpoint_tensor": "model.layers.17.self_attn.qkv_proj.weight", "logical_matrix": "L17.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 256 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.17.self_attn.o_proj.weight", "logical_matrix": "L17.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 696 ], "checkpoint_tensor": "model.layers.17.mlp.gate_up_proj.weight", "logical_matrix": "L17.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 696 ], "checkpoint_tensor": "model.layers.17.mlp.gate_up_proj.weight", "logical_matrix": "L17.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 696 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.17.mlp.down_proj.weight", "logical_matrix": "L17.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 256 ], "checkpoint_tensor": "model.layers.18.self_attn.qkv_proj.weight", "logical_matrix": "L18.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 256 ], "checkpoint_tensor": "model.layers.18.self_attn.qkv_proj.weight", "logical_matrix": "L18.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 256 ], "checkpoint_tensor": "model.layers.18.self_attn.qkv_proj.weight", "logical_matrix": "L18.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 256 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.18.self_attn.o_proj.weight", "logical_matrix": "L18.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 708 ], "checkpoint_tensor": "model.layers.18.mlp.gate_up_proj.weight", "logical_matrix": "L18.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 708 ], "checkpoint_tensor": "model.layers.18.mlp.gate_up_proj.weight", "logical_matrix": "L18.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 708 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.18.mlp.down_proj.weight", "logical_matrix": "L18.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 192 ], "checkpoint_tensor": "model.layers.19.self_attn.qkv_proj.weight", "logical_matrix": "L19.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 192 ], "checkpoint_tensor": "model.layers.19.self_attn.qkv_proj.weight", "logical_matrix": "L19.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 192 ], "checkpoint_tensor": "model.layers.19.self_attn.qkv_proj.weight", "logical_matrix": "L19.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 192 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.19.self_attn.o_proj.weight", "logical_matrix": "L19.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 446 ], "checkpoint_tensor": "model.layers.19.mlp.gate_up_proj.weight", "logical_matrix": "L19.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 446 ], "checkpoint_tensor": "model.layers.19.mlp.gate_up_proj.weight", "logical_matrix": "L19.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 446 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.19.mlp.down_proj.weight", "logical_matrix": "L19.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 256 ], "checkpoint_tensor": "model.layers.20.self_attn.qkv_proj.weight", "logical_matrix": "L20.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 256 ], "checkpoint_tensor": "model.layers.20.self_attn.qkv_proj.weight", "logical_matrix": "L20.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 256 ], "checkpoint_tensor": "model.layers.20.self_attn.qkv_proj.weight", "logical_matrix": "L20.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 256 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.20.self_attn.o_proj.weight", "logical_matrix": "L20.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 128 ], "checkpoint_tensor": "model.layers.20.mlp.gate_up_proj.weight", "logical_matrix": "L20.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 128 ], "checkpoint_tensor": "model.layers.20.mlp.gate_up_proj.weight", "logical_matrix": "L20.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 128 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.20.mlp.down_proj.weight", "logical_matrix": "L20.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 64 ], "checkpoint_tensor": "model.layers.21.self_attn.qkv_proj.weight", "logical_matrix": "L21.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 64 ], "checkpoint_tensor": "model.layers.21.self_attn.qkv_proj.weight", "logical_matrix": "L21.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 64 ], "checkpoint_tensor": "model.layers.21.self_attn.qkv_proj.weight", "logical_matrix": "L21.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 64 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.21.self_attn.o_proj.weight", "logical_matrix": "L21.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 113 ], "checkpoint_tensor": "model.layers.21.mlp.gate_up_proj.weight", "logical_matrix": "L21.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 113 ], "checkpoint_tensor": "model.layers.21.mlp.gate_up_proj.weight", "logical_matrix": "L21.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 113 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.21.mlp.down_proj.weight", "logical_matrix": "L21.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 64 ], "checkpoint_tensor": "model.layers.22.self_attn.qkv_proj.weight", "logical_matrix": "L22.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 64 ], "checkpoint_tensor": "model.layers.22.self_attn.qkv_proj.weight", "logical_matrix": "L22.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 64 ], "checkpoint_tensor": "model.layers.22.self_attn.qkv_proj.weight", "logical_matrix": "L22.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 64 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.22.self_attn.o_proj.weight", "logical_matrix": "L22.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 23 ], "checkpoint_tensor": "model.layers.22.mlp.gate_up_proj.weight", "logical_matrix": "L22.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 23 ], "checkpoint_tensor": "model.layers.22.mlp.gate_up_proj.weight", "logical_matrix": "L22.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 23 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.22.mlp.down_proj.weight", "logical_matrix": "L22.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 64 ], "checkpoint_tensor": "model.layers.23.self_attn.qkv_proj.weight", "logical_matrix": "L23.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 64 ], "checkpoint_tensor": "model.layers.23.self_attn.qkv_proj.weight", "logical_matrix": "L23.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 64 ], "checkpoint_tensor": "model.layers.23.self_attn.qkv_proj.weight", "logical_matrix": "L23.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 64 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.23.self_attn.o_proj.weight", "logical_matrix": "L23.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 9 ], "checkpoint_tensor": "model.layers.23.mlp.gate_up_proj.weight", "logical_matrix": "L23.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 9 ], "checkpoint_tensor": "model.layers.23.mlp.gate_up_proj.weight", "logical_matrix": "L23.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 9 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.23.mlp.down_proj.weight", "logical_matrix": "L23.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 128 ], "checkpoint_tensor": "model.layers.24.self_attn.qkv_proj.weight", "logical_matrix": "L24.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 128 ], "checkpoint_tensor": "model.layers.24.self_attn.qkv_proj.weight", "logical_matrix": "L24.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 128 ], "checkpoint_tensor": "model.layers.24.self_attn.qkv_proj.weight", "logical_matrix": "L24.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 128 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.24.self_attn.o_proj.weight", "logical_matrix": "L24.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 5 ], "checkpoint_tensor": "model.layers.24.mlp.gate_up_proj.weight", "logical_matrix": "L24.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 5 ], "checkpoint_tensor": "model.layers.24.mlp.gate_up_proj.weight", "logical_matrix": "L24.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 5 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.24.mlp.down_proj.weight", "logical_matrix": "L24.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 64 ], "checkpoint_tensor": "model.layers.25.self_attn.qkv_proj.weight", "logical_matrix": "L25.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 64 ], "checkpoint_tensor": "model.layers.25.self_attn.qkv_proj.weight", "logical_matrix": "L25.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 64 ], "checkpoint_tensor": "model.layers.25.self_attn.qkv_proj.weight", "logical_matrix": "L25.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 64 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.25.self_attn.o_proj.weight", "logical_matrix": "L25.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 47 ], "checkpoint_tensor": "model.layers.25.mlp.gate_up_proj.weight", "logical_matrix": "L25.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 47 ], "checkpoint_tensor": "model.layers.25.mlp.gate_up_proj.weight", "logical_matrix": "L25.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 47 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.25.mlp.down_proj.weight", "logical_matrix": "L25.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 64 ], "checkpoint_tensor": "model.layers.26.self_attn.qkv_proj.weight", "logical_matrix": "L26.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 64 ], "checkpoint_tensor": "model.layers.26.self_attn.qkv_proj.weight", "logical_matrix": "L26.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 64 ], "checkpoint_tensor": "model.layers.26.self_attn.qkv_proj.weight", "logical_matrix": "L26.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 64 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.26.self_attn.o_proj.weight", "logical_matrix": "L26.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 1963 ], "checkpoint_tensor": "model.layers.26.mlp.gate_up_proj.weight", "logical_matrix": "L26.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 1963 ], "checkpoint_tensor": "model.layers.26.mlp.gate_up_proj.weight", "logical_matrix": "L26.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 1963 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.26.mlp.down_proj.weight", "logical_matrix": "L26.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.27.self_attn.qkv_proj.weight", "logical_matrix": "L27.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 2048 ], "checkpoint_tensor": "model.layers.27.self_attn.qkv_proj.weight", "logical_matrix": "L27.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 2048 ], "checkpoint_tensor": "model.layers.27.self_attn.qkv_proj.weight", "logical_matrix": "L27.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.27.self_attn.o_proj.weight", "logical_matrix": "L27.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 3415 ], "checkpoint_tensor": "model.layers.27.mlp.gate_up_proj.weight", "logical_matrix": "L27.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 3415 ], "checkpoint_tensor": "model.layers.27.mlp.gate_up_proj.weight", "logical_matrix": "L27.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 3415 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.27.mlp.down_proj.weight", "logical_matrix": "L27.mlp.W_out", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 384 ], "checkpoint_tensor": "model.layers.28.self_attn.qkv_proj.weight", "logical_matrix": "L28.attn.W_Q", "scale": 8.0, "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 2048, 384 ], "checkpoint_tensor": "model.layers.28.self_attn.qkv_proj.weight", "logical_matrix": "L28.attn.W_K", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 4096, 384 ], "checkpoint_tensor": "model.layers.28.self_attn.qkv_proj.weight", "logical_matrix": "L28.attn.W_V", "transform": "transpose" }, { "checkpoint_columns": [ 0, 384 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.28.self_attn.o_proj.weight", "logical_matrix": "L28.attn.W_O", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 0, 307 ], "checkpoint_tensor": "model.layers.28.mlp.gate_up_proj.weight", "logical_matrix": "L28.mlp.W_in", "transform": "transpose" }, { "checkpoint_columns": [ 0, 2048 ], "checkpoint_rows": [ 3415, 307 ], "checkpoint_tensor": "model.layers.28.mlp.gate_up_proj.weight", "logical_matrix": "L28.mlp.W_up", "transform": "transpose" }, { "checkpoint_columns": [ 0, 307 ], "checkpoint_rows": [ 0, 2048 ], "checkpoint_tensor": "model.layers.28.mlp.down_proj.weight", "logical_matrix": "L28.mlp.W_out", "transform": "transpose" }, { "checkpoint_tensor": "model.embed_tokens.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.0.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.0.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.1.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.1.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.10.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.10.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.11.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.11.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.12.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.12.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.13.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.13.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.14.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.14.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.15.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.15.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.16.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.16.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.17.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.17.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.18.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.18.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.19.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.19.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.2.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.2.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.20.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.20.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.21.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.21.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.22.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.22.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.23.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.23.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.24.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.24.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.25.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.25.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.26.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.26.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.27.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.27.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.28.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.28.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.3.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.3.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.4.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.4.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.5.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.5.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.6.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.6.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.7.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.7.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.8.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.8.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.9.input_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.layers.9.post_attention_layernorm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" }, { "checkpoint_tensor": "model.norm.weight", "logical_matrix": null, "transform": "runtime_or_token_parameter" } ], "known_null_components": { "meaning": "These components are statically absent or padding. This is not an input-specific causal-relevance claim.", "never_referenced_residual_columns": [ [ 2033, 13 ] ], "padded_attention_heads": [ { "head_range": [ 1, 31 ], "layer": 0 }, { "head_range": [ 4, 28 ], "layer": 1 }, { "head_range": [ 3, 29 ], "layer": 2 }, { "head_range": [ 3, 29 ], "layer": 3 }, { "head_range": [ 8, 24 ], "layer": 4 }, { "head_range": [ 4, 28 ], "layer": 5 }, { "head_range": [ 2, 30 ], "layer": 6 }, { "head_range": [ 4, 28 ], "layer": 7 }, { "head_range": [ 5, 27 ], "layer": 8 }, { "head_range": [ 2, 30 ], "layer": 9 }, { "head_range": [ 4, 28 ], "layer": 10 }, { "head_range": [ 8, 24 ], "layer": 11 }, { "head_range": [ 12, 20 ], "layer": 12 }, { "head_range": [ 2, 30 ], "layer": 13 }, { "head_range": [ 2, 30 ], "layer": 14 }, { "head_range": [ 6, 26 ], "layer": 15 }, { "head_range": [ 4, 28 ], "layer": 16 }, { "head_range": [ 4, 28 ], "layer": 17 }, { "head_range": [ 4, 28 ], "layer": 18 }, { "head_range": [ 3, 29 ], "layer": 19 }, { "head_range": [ 4, 28 ], "layer": 20 }, { "head_range": [ 1, 31 ], "layer": 21 }, { "head_range": [ 1, 31 ], "layer": 22 }, { "head_range": [ 1, 31 ], "layer": 23 }, { "head_range": [ 2, 30 ], "layer": 24 }, { "head_range": [ 1, 31 ], "layer": 25 }, { "head_range": [ 1, 31 ], "layer": 26 }, { "head_range": [ 6, 26 ], "layer": 28 } ], "padded_mlp_neurons": [ { "layer": 0, "neuron_range": [ 16, 3399 ] }, { "layer": 1, "neuron_range": [ 29, 3386 ] }, { "layer": 2, "neuron_range": [ 73, 3342 ] }, { "layer": 3, "neuron_range": [ 25, 3390 ] }, { "layer": 4, "neuron_range": [ 376, 3039 ] }, { "layer": 5, "neuron_range": [ 364, 3051 ] }, { "layer": 6, "neuron_range": [ 82, 3333 ] }, { "layer": 7, "neuron_range": [ 54, 3361 ] }, { "layer": 8, "neuron_range": [ 6, 3409 ] }, { "layer": 9, "neuron_range": [ 1, 3414 ] }, { "layer": 10, "neuron_range": [ 3, 3412 ] }, { "layer": 11, "neuron_range": [ 1543, 1872 ] }, { "layer": 12, "neuron_range": [ 399, 3016 ] }, { "layer": 13, "neuron_range": [ 1888, 1527 ] }, { "layer": 14, "neuron_range": [ 2254, 1161 ] }, { "layer": 15, "neuron_range": [ 1274, 2141 ] }, { "layer": 16, "neuron_range": [ 792, 2623 ] }, { "layer": 17, "neuron_range": [ 696, 2719 ] }, { "layer": 18, "neuron_range": [ 708, 2707 ] }, { "layer": 19, "neuron_range": [ 446, 2969 ] }, { "layer": 20, "neuron_range": [ 128, 3287 ] }, { "layer": 21, "neuron_range": [ 113, 3302 ] }, { "layer": 22, "neuron_range": [ 23, 3392 ] }, { "layer": 23, "neuron_range": [ 9, 3406 ] }, { "layer": 24, "neuron_range": [ 5, 3410 ] }, { "layer": 25, "neuron_range": [ 47, 3368 ] }, { "layer": 26, "neuron_range": [ 1963, 1452 ] }, { "layer": 28, "neuron_range": [ 307, 3108 ] } ] }, "matrices": { "L0.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 64 ] }, "L0.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 64, 2048 ] }, "L0.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 64 ] }, "L0.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 64 ] }, "L0.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 16 ] }, "L0.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 16, 2048 ] }, "L0.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 16 ] }, "L1.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 256 ] }, "L1.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 256, 2048 ] }, "L1.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 256 ] }, "L1.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 256 ] }, "L1.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 29 ] }, "L1.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 29, 2048 ] }, "L1.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 29 ] }, "L10.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 256 ] }, "L10.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 256, 2048 ] }, "L10.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 256 ] }, "L10.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 256 ] }, "L10.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 3 ] }, "L10.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 3, 2048 ] }, "L10.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 3 ] }, "L11.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 512 ] }, "L11.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 512, 2048 ] }, "L11.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 512 ] }, "L11.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 512 ] }, "L11.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 1543 ] }, "L11.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 1543, 2048 ] }, "L11.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 1543 ] }, "L12.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 768 ] }, "L12.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 768, 2048 ] }, "L12.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 768 ] }, "L12.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 768 ] }, "L12.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 399 ] }, "L12.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 399, 2048 ] }, "L12.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 399 ] }, "L13.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 128 ] }, "L13.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 128, 2048 ] }, "L13.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 128 ] }, "L13.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 128 ] }, "L13.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 1888 ] }, "L13.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 1888, 2048 ] }, "L13.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 1888 ] }, "L14.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 128 ] }, "L14.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 128, 2048 ] }, "L14.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 128 ] }, "L14.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 128 ] }, "L14.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 2254 ] }, "L14.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 2254, 2048 ] }, "L14.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 2254 ] }, "L15.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 384 ] }, "L15.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 384, 2048 ] }, "L15.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 384 ] }, "L15.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 384 ] }, "L15.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 1274 ] }, "L15.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 1274, 2048 ] }, "L15.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 1274 ] }, "L16.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 256 ] }, "L16.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 256, 2048 ] }, "L16.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 256 ] }, "L16.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 256 ] }, "L16.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 792 ] }, "L16.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 792, 2048 ] }, "L16.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 792 ] }, "L17.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 256 ] }, "L17.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 256, 2048 ] }, "L17.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 256 ] }, "L17.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 256 ] }, "L17.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 696 ] }, "L17.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 696, 2048 ] }, "L17.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 696 ] }, "L18.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 256 ] }, "L18.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 256, 2048 ] }, "L18.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 256 ] }, "L18.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 256 ] }, "L18.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 708 ] }, "L18.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 708, 2048 ] }, "L18.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 708 ] }, "L19.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 192 ] }, "L19.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 192, 2048 ] }, "L19.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 192 ] }, "L19.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 192 ] }, "L19.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 446 ] }, "L19.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 446, 2048 ] }, "L19.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 446 ] }, "L2.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 192 ] }, "L2.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 192, 2048 ] }, "L2.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 192 ] }, "L2.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 192 ] }, "L2.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 73 ] }, "L2.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 73, 2048 ] }, "L2.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 73 ] }, "L20.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 256 ] }, "L20.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 256, 2048 ] }, "L20.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 256 ] }, "L20.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 256 ] }, "L20.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 128 ] }, "L20.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 128, 2048 ] }, "L20.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 128 ] }, "L21.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 64 ] }, "L21.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 64, 2048 ] }, "L21.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 64 ] }, "L21.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 64 ] }, "L21.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 113 ] }, "L21.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 113, 2048 ] }, "L21.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 113 ] }, "L22.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 64 ] }, "L22.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 64, 2048 ] }, "L22.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 64 ] }, "L22.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 64 ] }, "L22.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 23 ] }, "L22.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 23, 2048 ] }, "L22.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 23 ] }, "L23.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 64 ] }, "L23.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 64, 2048 ] }, "L23.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 64 ] }, "L23.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 64 ] }, "L23.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 9 ] }, "L23.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 9, 2048 ] }, "L23.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 9 ] }, "L24.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 128 ] }, "L24.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 128, 2048 ] }, "L24.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 128 ] }, "L24.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 128 ] }, "L24.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 5 ] }, "L24.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 5, 2048 ] }, "L24.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 5 ] }, "L25.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 64 ] }, "L25.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 64, 2048 ] }, "L25.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 64 ] }, "L25.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 64 ] }, "L25.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 47 ] }, "L25.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 47, 2048 ] }, "L25.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 47 ] }, "L26.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 64 ] }, "L26.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 64, 2048 ] }, "L26.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 64 ] }, "L26.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 64 ] }, "L26.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 1963 ] }, "L26.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 1963, 2048 ] }, "L26.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 1963 ] }, "L27.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 2048 ] }, "L27.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 2048, 2048 ] }, "L27.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 2048 ] }, "L27.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 2048 ] }, "L27.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 3415 ] }, "L27.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 3415, 2048 ] }, "L27.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 3415 ] }, "L28.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 384 ] }, "L28.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 384, 2048 ] }, "L28.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 384 ] }, "L28.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 384 ] }, "L28.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 307 ] }, "L28.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 307, 2048 ] }, "L28.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 307 ] }, "L3.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 192 ] }, "L3.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 192, 2048 ] }, "L3.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 192 ] }, "L3.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 192 ] }, "L3.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 25 ] }, "L3.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 25, 2048 ] }, "L3.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 25 ] }, "L4.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 512 ] }, "L4.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 512, 2048 ] }, "L4.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 512 ] }, "L4.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 512 ] }, "L4.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 376 ] }, "L4.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 376, 2048 ] }, "L4.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 376 ] }, "L5.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 256 ] }, "L5.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 256, 2048 ] }, "L5.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 256 ] }, "L5.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 256 ] }, "L5.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 364 ] }, "L5.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 364, 2048 ] }, "L5.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 364 ] }, "L6.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 128 ] }, "L6.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 128, 2048 ] }, "L6.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 128 ] }, "L6.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 128 ] }, "L6.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 82 ] }, "L6.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 82, 2048 ] }, "L6.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 82 ] }, "L7.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 256 ] }, "L7.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 256, 2048 ] }, "L7.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 256 ] }, "L7.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 256 ] }, "L7.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 54 ] }, "L7.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 54, 2048 ] }, "L7.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 54 ] }, "L8.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 320 ] }, "L8.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 320, 2048 ] }, "L8.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 320 ] }, "L8.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 320 ] }, "L8.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 6 ] }, "L8.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 6, 2048 ] }, "L8.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 6 ] }, "L9.attn.W_K": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 128 ] }, "L9.attn.W_O": { "axis0": "head", "axis1": "residual_out", "shape": [ 128, 2048 ] }, "L9.attn.W_Q": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 128 ] }, "L9.attn.W_V": { "axis0": "residual_in", "axis1": "head", "shape": [ 2048, 128 ] }, "L9.mlp.W_in": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 1 ] }, "L9.mlp.W_out": { "axis0": "hidden", "axis1": "residual_out", "shape": [ 1, 2048 ] }, "L9.mlp.W_up": { "axis0": "residual_in", "axis1": "hidden", "shape": [ 2048, 1 ] } }, "matrix_coordinates": "logical x@W compiler coordinates", "placements": { "i:0": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L28.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L28.mlp.W_out" } ], "l:0": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 34 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 34 ], "matrix": "L28.mlp.W_up" }, { "axis0": [ 18, 1 ], "axis1": [ 1, 34 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 26, 2 ], "axis1": [ 1, 34 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 65, 1 ], "axis1": [ 1, 34 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 82, 8 ], "axis1": [ 1, 34 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 91, 2 ], "axis1": [ 1, 34 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 329, 1 ], "axis1": [ 1, 34 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 1631, 1 ], "axis1": [ 1, 34 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 1645, 2 ], "axis1": [ 1, 34 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 2016, 17 ], "axis1": [ 1, 34 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 18, 1 ], "axis1": [ 1, 34 ], "matrix": "L28.mlp.W_up" }, { "axis0": [ 26, 2 ], "axis1": [ 1, 34 ], "matrix": "L28.mlp.W_up" }, { "axis0": [ 65, 1 ], "axis1": [ 1, 34 ], "matrix": "L28.mlp.W_up" }, { "axis0": [ 82, 8 ], "axis1": [ 1, 34 ], "matrix": "L28.mlp.W_up" }, { "axis0": [ 91, 2 ], "axis1": [ 1, 34 ], "matrix": "L28.mlp.W_up" }, { "axis0": [ 329, 1 ], "axis1": [ 1, 34 ], "matrix": "L28.mlp.W_up" }, { "axis0": [ 1631, 1 ], "axis1": [ 1, 34 ], "matrix": "L28.mlp.W_up" }, { "axis0": [ 1645, 2 ], "axis1": [ 1, 34 ], "matrix": "L28.mlp.W_up" }, { "axis0": [ 2016, 17 ], "axis1": [ 1, 34 ], "matrix": "L28.mlp.W_up" }, { "axis0": [ 1, 34 ], "axis1": [ 1, 17 ], "matrix": "L28.mlp.W_out" } ], "l:10": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L28.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L28.attn.W_K" }, { "axis0": [ 44, 1 ], "axis1": [ 0, 64 ], "matrix": "L28.attn.W_V" }, { "axis0": [ 57, 1 ], "axis1": [ 0, 64 ], "matrix": "L28.attn.W_V" }, { "axis0": [ 74, 5 ], "axis1": [ 0, 64 ], "matrix": "L28.attn.W_V" }, { "axis0": [ 80, 1 ], "axis1": [ 0, 64 ], "matrix": "L28.attn.W_V" }, { "axis0": [ 217, 4 ], "axis1": [ 0, 64 ], "matrix": "L28.attn.W_V" }, { "axis0": [ 222, 1 ], "axis1": [ 0, 64 ], "matrix": "L28.attn.W_V" }, { "axis0": [ 274, 11 ], "axis1": [ 0, 64 ], "matrix": "L28.attn.W_V" }, { "axis0": [ 288, 2 ], "axis1": [ 0, 64 ], "matrix": "L28.attn.W_V" }, { "axis0": [ 302, 1 ], "axis1": [ 0, 64 ], "matrix": "L28.attn.W_V" }, { "axis0": [ 337, 1 ], "axis1": [ 0, 64 ], "matrix": "L28.attn.W_V" }, { "axis0": [ 339, 1 ], "axis1": [ 0, 64 ], "matrix": "L28.attn.W_V" }, { "axis0": [ 1909, 35 ], "axis1": [ 0, 64 ], "matrix": "L28.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 18, 1 ], "matrix": "L28.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 27, 1 ], "matrix": "L28.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 65, 1 ], "matrix": "L28.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 82, 8 ], "matrix": "L28.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 91, 2 ], "matrix": "L28.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 329, 1 ], "matrix": "L28.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 1631, 1 ], "matrix": "L28.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 1645, 2 ], "matrix": "L28.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L28.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L28.attn.W_K" }, { "axis0": [ 1944, 64 ], "axis1": [ 64, 64 ], "matrix": "L28.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 18, 1 ], "matrix": "L28.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 27, 1 ], "matrix": "L28.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 65, 1 ], "matrix": "L28.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 82, 8 ], "matrix": "L28.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 91, 2 ], "matrix": "L28.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 329, 1 ], "matrix": "L28.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 1631, 1 ], "matrix": "L28.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 1645, 2 ], "matrix": "L28.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L28.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L28.attn.W_K" }, { "axis0": [ 2008, 8 ], "axis1": [ 128, 64 ], "matrix": "L28.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 18, 1 ], "matrix": "L28.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 27, 1 ], "matrix": "L28.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 65, 1 ], "matrix": "L28.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 82, 8 ], "matrix": "L28.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 91, 2 ], "matrix": "L28.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 329, 1 ], "matrix": "L28.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 1631, 1 ], "matrix": "L28.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 1645, 2 ], "matrix": "L28.attn.W_O" } ], "l:100": [ { "axis0": [ 0, 1 ], "axis1": [ 33, 1 ], "matrix": "L0.mlp.W_out" } ], "l:101": [ { "axis0": [ 27, 2 ], "axis1": [ 64, 64 ], "matrix": "L1.attn.W_Q" }, { "axis0": [ 22, 1 ], "axis1": [ 64, 64 ], "matrix": "L1.attn.W_K" }, { "axis0": [ 29, 1 ], "axis1": [ 64, 64 ], "matrix": "L1.attn.W_K" }, { "axis0": [ 22, 1 ], "axis1": [ 64, 64 ], "matrix": "L1.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 25, 1 ], "matrix": "L1.attn.W_O" } ], "l:103": [ { "axis0": [ 0, 1 ], "axis1": [ 27, 1 ], "matrix": "L0.mlp.W_out" } ], "l:104": [ { "axis0": [ 0, 1 ], "axis1": [ 28, 1 ], "matrix": "L0.mlp.W_out" } ], "l:106": [ { "axis0": [ 0, 1 ], "axis1": [ 5, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 5, 1 ], "matrix": "L0.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 22, 1 ], "matrix": "L0.mlp.W_out" }, { "axis0": [ 1, 17 ], "axis1": [ 5, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 5, 1 ], "axis1": [ 22, 1 ], "matrix": "L0.mlp.W_out" } ], "l:107": [ { "axis0": [ 0, 1 ], "axis1": [ 29, 1 ], "matrix": "L0.mlp.W_out" } ], "l:108": [ { "axis0": [ 0, 1 ], "axis1": [ 15, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 22, 1 ], "axis1": [ 15, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 22, 1 ], "axis1": [ 16, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 15, 1 ], "axis1": [ 46, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 16, 1 ], "axis1": [ 46, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 46, 1 ], "matrix": "L1.mlp.W_out" } ], "l:109": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L7.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L7.attn.W_K" }, { "axis0": [ 23, 1 ], "axis1": [ 0, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 52, 1 ], "axis1": [ 0, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 102, 1 ], "axis1": [ 0, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 210, 1 ], "axis1": [ 0, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 42, 4 ], "matrix": "L7.attn.W_O" } ], "l:111": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 3 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 3 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 103, 1 ], "axis1": [ 1, 3 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 236, 3 ], "axis1": [ 1, 3 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 103, 1 ], "axis1": [ 1, 3 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 236, 3 ], "axis1": [ 1, 3 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1, 3 ], "axis1": [ 23, 1 ], "matrix": "L6.mlp.W_out" }, { "axis0": [ 1, 3 ], "axis1": [ 52, 1 ], "matrix": "L6.mlp.W_out" }, { "axis0": [ 1, 3 ], "axis1": [ 102, 1 ], "matrix": "L6.mlp.W_out" } ], "l:113": [ { "axis0": [ 0, 1 ], "axis1": [ 33, 2 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 33, 2 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 103, 1 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 23, 1 ], "axis1": [ 33, 2 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 51, 1 ], "axis1": [ 33, 2 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 33, 2 ], "axis1": [ 103, 1 ], "matrix": "L5.mlp.W_out" } ], "l:115": [ { "axis0": [ 0, 1 ], "axis1": [ 6, 10 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 6, 10 ], "matrix": "L0.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 23, 1 ], "matrix": "L0.mlp.W_out" }, { "axis0": [ 1, 17 ], "axis1": [ 6, 10 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 6, 10 ], "axis1": [ 23, 1 ], "matrix": "L0.mlp.W_out" } ], "l:116": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 51, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 31, 1 ], "axis1": [ 1, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 1, 2 ], "axis1": [ 51, 1 ], "matrix": "L4.mlp.W_out" } ], "l:117": [ { "axis0": [ 50, 2 ], "axis1": [ 64, 64 ], "matrix": "L4.attn.W_Q" }, { "axis0": [ 27, 1 ], "axis1": [ 64, 64 ], "matrix": "L4.attn.W_K" }, { "axis0": [ 52, 1 ], "axis1": [ 64, 64 ], "matrix": "L4.attn.W_K" }, { "axis0": [ 27, 1 ], "axis1": [ 64, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 31, 1 ], "matrix": "L4.attn.W_O" } ], "l:119": [ { "axis0": [ 0, 1 ], "axis1": [ 50, 1 ], "matrix": "L3.mlp.W_out" } ], "l:12": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 18, 1 ], "axis1": [ 1, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 65, 1 ], "axis1": [ 1, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 81, 13 ], "axis1": [ 1, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 285, 1 ], "axis1": [ 1, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 326, 1 ], "axis1": [ 1, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 329, 1 ], "axis1": [ 1, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 18, 1 ], "axis1": [ 1, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 65, 1 ], "axis1": [ 1, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 81, 13 ], "axis1": [ 1, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 285, 1 ], "axis1": [ 1, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 326, 1 ], "axis1": [ 1, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 329, 1 ], "axis1": [ 1, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 1, 17 ], "axis1": [ 44, 1 ], "matrix": "L27.mlp.W_out" }, { "axis0": [ 1, 17 ], "axis1": [ 74, 3 ], "matrix": "L27.mlp.W_out" }, { "axis0": [ 1, 17 ], "axis1": [ 80, 1 ], "matrix": "L27.mlp.W_out" }, { "axis0": [ 1, 17 ], "axis1": [ 217, 4 ], "matrix": "L27.mlp.W_out" }, { "axis0": [ 1, 17 ], "axis1": [ 222, 1 ], "matrix": "L27.mlp.W_out" }, { "axis0": [ 1, 17 ], "axis1": [ 274, 7 ], "matrix": "L27.mlp.W_out" } ], "l:120": [ { "axis0": [ 0, 1 ], "axis1": [ 51, 1 ], "matrix": "L3.mlp.W_out" } ], "l:122": [ { "axis0": [ 0, 1 ], "axis1": [ 52, 1 ], "matrix": "L3.mlp.W_out" } ], "l:123": [ { "axis0": [ 0, 1 ], "axis1": [ 130, 6 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 52, 1 ], "axis1": [ 130, 3 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 57, 2 ], "axis1": [ 130, 3 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 52, 1 ], "axis1": [ 133, 3 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 57, 2 ], "axis1": [ 133, 3 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 130, 3 ], "axis1": [ 236, 3 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 133, 3 ], "axis1": [ 236, 3 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 236, 3 ], "matrix": "L5.mlp.W_out" } ], "l:124": [ { "axis0": [ 0, 1 ], "axis1": [ 3, 12 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 3, 12 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 3, 12 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 52, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 57, 2 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 49, 2 ], "axis1": [ 3, 12 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 3, 12 ], "axis1": [ 52, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 3, 12 ], "axis1": [ 57, 2 ], "matrix": "L4.mlp.W_out" } ], "l:126": [ { "axis0": [ 0, 1 ], "axis1": [ 15, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 22, 1 ], "axis1": [ 15, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 22, 1 ], "axis1": [ 16, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 15, 1 ], "axis1": [ 49, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 16, 1 ], "axis1": [ 49, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 15, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 16, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 49, 1 ], "matrix": "L3.mlp.W_out" } ], "l:127": [ { "axis0": [ 0, 1 ], "axis1": [ 384, 64 ], "matrix": "L4.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 384, 64 ], "matrix": "L4.attn.W_K" }, { "axis0": [ 49, 1 ], "axis1": [ 384, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 384, 64 ], "axis1": [ 50, 1 ], "matrix": "L4.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 50, 1 ], "matrix": "L4.mlp.W_out" } ], "l:129": [ { "axis0": [ 0, 1 ], "axis1": [ 11, 4 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 11, 4 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 22, 1 ], "axis1": [ 11, 4 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 35, 4 ], "axis1": [ 11, 4 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 22, 1 ], "axis1": [ 11, 4 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 35, 4 ], "axis1": [ 11, 4 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 11, 4 ], "axis1": [ 42, 4 ], "matrix": "L1.mlp.W_out" } ], "l:131": [ { "axis0": [ 0, 1 ], "axis1": [ 35, 4 ], "matrix": "L0.mlp.W_out" } ], "l:132": [ { "axis0": [ 0, 1 ], "axis1": [ 4, 34 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 4, 34 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 1, 17 ], "axis1": [ 4, 34 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 49, 2 ], "axis1": [ 4, 34 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 53, 4 ], "axis1": [ 4, 34 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 103, 1 ], "axis1": [ 4, 34 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 239, 2 ], "axis1": [ 4, 34 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 288, 1 ], "axis1": [ 4, 34 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 323, 8 ], "axis1": [ 4, 34 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 1, 17 ], "axis1": [ 4, 34 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 49, 2 ], "axis1": [ 4, 34 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 53, 4 ], "axis1": [ 4, 34 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 103, 1 ], "axis1": [ 4, 34 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 239, 2 ], "axis1": [ 4, 34 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 288, 1 ], "axis1": [ 4, 34 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 323, 8 ], "axis1": [ 4, 34 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 4, 34 ], "axis1": [ 24, 1 ], "matrix": "L6.mlp.W_out" }, { "axis0": [ 4, 34 ], "axis1": [ 173, 16 ], "matrix": "L6.mlp.W_out" } ], "l:134": [ { "axis0": [ 0, 1 ], "axis1": [ 49, 2 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 53, 4 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 239, 2 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 288, 1 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 323, 8 ], "matrix": "L5.mlp.W_out" } ], "l:135": [ { "axis0": [ 263, 2 ], "axis1": [ 256, 64 ], "matrix": "L11.attn.W_Q" }, { "axis0": [ 21, 1 ], "axis1": [ 256, 64 ], "matrix": "L11.attn.W_K" }, { "axis0": [ 265, 1 ], "axis1": [ 256, 64 ], "matrix": "L11.attn.W_K" }, { "axis0": [ 111, 10 ], "axis1": [ 256, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 207, 3 ], "axis1": [ 256, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 211, 2 ], "axis1": [ 256, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 222, 1 ], "axis1": [ 256, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 327, 1 ], "axis1": [ 256, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 256, 64 ], "axis1": [ 49, 2 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 239, 8 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 248, 3 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 252, 4 ], "matrix": "L11.attn.W_O" } ], "l:137": [ { "axis0": [ 0, 1 ], "axis1": [ 263, 1 ], "matrix": "L10.mlp.W_out" } ], "l:138": [ { "axis0": [ 0, 1 ], "axis1": [ 264, 1 ], "matrix": "L10.mlp.W_out" } ], "l:14": [ { "axis0": [ 0, 1 ], "axis1": [ 216, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 216, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 285, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 28, 2 ], "axis1": [ 216, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 216, 2 ], "axis1": [ 285, 1 ], "matrix": "L4.mlp.W_out" } ], "l:140": [ { "axis0": [ 0, 1 ], "axis1": [ 265, 1 ], "matrix": "L10.mlp.W_out" } ], "l:141": [ { "axis0": [ 208, 2 ], "axis1": [ 64, 64 ], "matrix": "L10.attn.W_Q" }, { "axis0": [ 211, 2 ], "axis1": [ 64, 64 ], "matrix": "L10.attn.W_Q" }, { "axis0": [ 60, 4 ], "axis1": [ 64, 64 ], "matrix": "L10.attn.W_K" }, { "axis0": [ 104, 17 ], "axis1": [ 64, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 111, 10 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 207, 3 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 211, 2 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 222, 1 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 327, 1 ], "matrix": "L10.attn.W_O" } ], "l:142": [ { "axis0": [ 0, 1 ], "axis1": [ 43, 9 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 43, 9 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 208, 2 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 211, 2 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 326, 1 ], "axis1": [ 43, 9 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 43, 9 ], "axis1": [ 208, 2 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 43, 9 ], "axis1": [ 211, 2 ], "matrix": "L7.mlp.W_out" } ], "l:143": [ { "axis0": [ 207, 2 ], "axis1": [ 64, 64 ], "matrix": "L7.attn.W_Q" }, { "axis0": [ 21, 1 ], "axis1": [ 64, 64 ], "matrix": "L7.attn.W_K" }, { "axis0": [ 209, 1 ], "axis1": [ 64, 64 ], "matrix": "L7.attn.W_K" }, { "axis0": [ 51, 1 ], "axis1": [ 64, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 326, 1 ], "matrix": "L7.attn.W_O" } ], "l:145": [ { "axis0": [ 0, 1 ], "axis1": [ 207, 1 ], "matrix": "L6.mlp.W_out" } ], "l:146": [ { "axis0": [ 0, 1 ], "axis1": [ 208, 1 ], "matrix": "L6.mlp.W_out" } ], "l:148": [ { "axis0": [ 0, 1 ], "axis1": [ 209, 1 ], "matrix": "L6.mlp.W_out" } ], "l:149": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L6.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L6.attn.W_K" }, { "axis0": [ 102, 1 ], "axis1": [ 0, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 51, 1 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 51, 1 ], "matrix": "L6.mlp.W_out" } ], "l:150": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 32 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 32 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 102, 1 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 24, 1 ], "axis1": [ 1, 32 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1, 32 ], "axis1": [ 102, 1 ], "matrix": "L5.mlp.W_out" } ], "l:151": [ { "axis0": [ 34, 1 ], "axis1": [ 0, 64 ], "matrix": "L5.attn.W_Q" }, { "axis0": [ 30, 1 ], "axis1": [ 0, 64 ], "matrix": "L5.attn.W_K" }, { "axis0": [ 288, 1 ], "axis1": [ 0, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 24, 1 ], "matrix": "L5.attn.W_O" } ], "l:152": [ { "axis0": [ 0, 1 ], "axis1": [ 34, 1 ], "matrix": "L0.mlp.W_out" } ], "l:153": [ { "axis0": [ 30, 2 ], "axis1": [ 0, 64 ], "matrix": "L4.attn.W_Q" }, { "axis0": [ 27, 1 ], "axis1": [ 0, 64 ], "matrix": "L4.attn.W_K" }, { "axis0": [ 32, 1 ], "axis1": [ 0, 64 ], "matrix": "L4.attn.W_K" }, { "axis0": [ 27, 1 ], "axis1": [ 0, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 30, 1 ], "matrix": "L4.attn.W_O" } ], "l:155": [ { "axis0": [ 0, 1 ], "axis1": [ 30, 1 ], "matrix": "L0.mlp.W_out" } ], "l:156": [ { "axis0": [ 0, 1 ], "axis1": [ 31, 1 ], "matrix": "L0.mlp.W_out" } ], "l:158": [ { "axis0": [ 0, 1 ], "axis1": [ 32, 1 ], "matrix": "L0.mlp.W_out" } ], "l:159": [ { "axis0": [ 0, 1 ], "axis1": [ 222, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 27, 1 ], "axis1": [ 222, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 27, 1 ], "axis1": [ 223, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 222, 1 ], "axis1": [ 288, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 223, 1 ], "axis1": [ 288, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 288, 1 ], "matrix": "L4.mlp.W_out" } ], "l:16": [ { "axis0": [ 0, 1 ], "axis1": [ 3, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 3, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 3, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 28, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 24, 1 ], "axis1": [ 3, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 3, 2 ], "axis1": [ 28, 1 ], "matrix": "L3.mlp.W_out" } ], "l:160": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L9.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L9.attn.W_K" }, { "axis0": [ 49, 1 ], "axis1": [ 0, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 236, 3 ], "axis1": [ 0, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 60, 4 ], "matrix": "L9.attn.W_O" } ], "l:162": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 3 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 3 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 3 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 3 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 45, 1 ], "axis1": [ 1, 3 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 59, 1 ], "axis1": [ 1, 3 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 222, 2 ], "axis1": [ 1, 3 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 45, 1 ], "axis1": [ 1, 3 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 59, 1 ], "axis1": [ 1, 3 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 222, 2 ], "axis1": [ 1, 3 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 1, 3 ], "axis1": [ 236, 3 ], "matrix": "L8.mlp.W_out" } ], "l:164": [ { "axis0": [ 0, 1 ], "axis1": [ 15, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 15, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 59, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 23, 1 ], "axis1": [ 15, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 31, 1 ], "axis1": [ 15, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 39, 1 ], "axis1": [ 15, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 15, 2 ], "axis1": [ 59, 1 ], "matrix": "L4.mlp.W_out" } ], "l:166": [ { "axis0": [ 0, 1 ], "axis1": [ 9, 2 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 9, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 39, 1 ], "matrix": "L1.mlp.W_out" }, { "axis0": [ 24, 1 ], "axis1": [ 9, 2 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 9, 2 ], "axis1": [ 39, 1 ], "matrix": "L1.mlp.W_out" } ], "l:167": [ { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L8.attn.W_K" }, { "axis0": [ 51, 1 ], "axis1": [ 192, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 53, 2 ], "axis1": [ 192, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 192, 64 ], "axis1": [ 45, 1 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 222, 2 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 45, 1 ], "matrix": "L8.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 222, 2 ], "matrix": "L8.mlp.W_out" } ], "l:168": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 12 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 12 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 12 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 51, 1 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 53, 2 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 50, 1 ], "axis1": [ 1, 12 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 189, 1 ], "axis1": [ 1, 12 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 1, 12 ], "axis1": [ 51, 1 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 1, 12 ], "axis1": [ 53, 2 ], "matrix": "L7.mlp.W_out" } ], "l:17": [ { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L3.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L3.attn.W_K" }, { "axis0": [ 35, 1 ], "axis1": [ 64, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 24, 1 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 24, 1 ], "matrix": "L3.mlp.W_out" } ], "l:170": [ { "axis0": [ 0, 1 ], "axis1": [ 38, 2 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 51, 1 ], "axis1": [ 38, 1 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 51, 1 ], "axis1": [ 39, 1 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 38, 1 ], "axis1": [ 189, 1 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 39, 1 ], "axis1": [ 189, 1 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 38, 1 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 39, 1 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 189, 1 ], "matrix": "L6.mlp.W_out" } ], "l:171": [ { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L7.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L7.attn.W_K" }, { "axis0": [ 189, 1 ], "axis1": [ 128, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 50, 1 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 50, 1 ], "matrix": "L7.mlp.W_out" } ], "l:173": [ { "axis0": [ 0, 1 ], "axis1": [ 17, 4 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 17, 4 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 27, 1 ], "axis1": [ 17, 4 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 53, 4 ], "axis1": [ 17, 4 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 27, 1 ], "axis1": [ 17, 4 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 53, 4 ], "axis1": [ 17, 4 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 17, 4 ], "axis1": [ 60, 4 ], "matrix": "L4.mlp.W_out" } ], "l:175": [ { "axis0": [ 0, 1 ], "axis1": [ 53, 4 ], "matrix": "L3.mlp.W_out" } ], "l:176": [ { "axis0": [ 0, 1 ], "axis1": [ 35, 34 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 35, 34 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 1, 17 ], "axis1": [ 35, 34 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 59, 1 ], "axis1": [ 35, 34 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 289, 17 ], "axis1": [ 35, 34 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 1, 17 ], "axis1": [ 35, 34 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 59, 1 ], "axis1": [ 35, 34 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 289, 17 ], "axis1": [ 35, 34 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 35, 34 ], "axis1": [ 104, 17 ], "matrix": "L5.mlp.W_out" } ], "l:178": [ { "axis0": [ 0, 1 ], "axis1": [ 289, 17 ], "matrix": "L4.mlp.W_out" } ], "l:179": [ { "axis0": [ 0, 1 ], "axis1": [ 46, 42 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 46, 42 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 341, 1 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 343, 1 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 377, 12 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 390, 1 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 394, 1 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 398, 1 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 208, 1 ], "axis1": [ 46, 42 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 46, 42 ], "axis1": [ 341, 1 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 46, 42 ], "axis1": [ 343, 1 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 46, 42 ], "axis1": [ 377, 12 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 46, 42 ], "axis1": [ 390, 1 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 46, 42 ], "axis1": [ 394, 1 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 46, 42 ], "axis1": [ 398, 1 ], "matrix": "L16.mlp.W_out" } ], "l:18": [ { "axis0": [ 0, 1 ], "axis1": [ 35, 32 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 35, 32 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 35, 1 ], "matrix": "L2.mlp.W_out" }, { "axis0": [ 24, 1 ], "axis1": [ 35, 32 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 35, 32 ], "axis1": [ 35, 1 ], "matrix": "L2.mlp.W_out" } ], "l:180": [ { "axis0": [ 0, 1 ], "axis1": [ 448, 64 ], "matrix": "L12.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 448, 64 ], "matrix": "L12.attn.W_K" }, { "axis0": [ 308, 11 ], "axis1": [ 448, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 320, 3 ], "axis1": [ 448, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 331, 50 ], "axis1": [ 448, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 448, 64 ], "axis1": [ 208, 1 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 512, 64 ], "matrix": "L12.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 512, 64 ], "matrix": "L12.attn.W_K" }, { "axis0": [ 381, 4 ], "axis1": [ 512, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 419, 34 ], "axis1": [ 512, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 512, 64 ], "axis1": [ 208, 1 ], "matrix": "L12.attn.W_O" } ], "l:182": [ { "axis0": [ 0, 1 ], "axis1": [ 201, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 201, 100 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 351, 34 ], "matrix": "L11.mlp.W_out" }, { "axis0": [ 51, 4 ], "axis1": [ 201, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 67, 1 ], "axis1": [ 201, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 174, 16 ], "axis1": [ 201, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 210, 1 ], "axis1": [ 201, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 258, 5 ], "axis1": [ 201, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 266, 7 ], "axis1": [ 201, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 201, 100 ], "axis1": [ 351, 34 ], "matrix": "L11.mlp.W_out" } ], "l:184": [ { "axis0": [ 266, 2 ], "axis1": [ 320, 64 ], "matrix": "L11.attn.W_Q" }, { "axis0": [ 21, 1 ], "axis1": [ 320, 64 ], "matrix": "L11.attn.W_K" }, { "axis0": [ 268, 1 ], "axis1": [ 320, 64 ], "matrix": "L11.attn.W_K" }, { "axis0": [ 45, 1 ], "axis1": [ 320, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 223, 1 ], "axis1": [ 320, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 236, 3 ], "axis1": [ 320, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 240, 12 ], "axis1": [ 320, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 320, 64 ], "axis1": [ 51, 4 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 67, 1 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 258, 5 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 266, 7 ], "matrix": "L11.attn.W_O" } ], "l:186": [ { "axis0": [ 0, 1 ], "axis1": [ 266, 1 ], "matrix": "L10.mlp.W_out" } ], "l:187": [ { "axis0": [ 0, 1 ], "axis1": [ 267, 1 ], "matrix": "L10.mlp.W_out" } ], "l:189": [ { "axis0": [ 0, 1 ], "axis1": [ 268, 1 ], "matrix": "L10.mlp.W_out" } ], "l:19": [ { "axis0": [ 48, 1 ], "axis1": [ 64, 64 ], "matrix": "L2.attn.W_Q" }, { "axis0": [ 26, 1 ], "axis1": [ 64, 64 ], "matrix": "L2.attn.W_K" }, { "axis0": [ 47, 1 ], "axis1": [ 64, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 24, 1 ], "matrix": "L2.attn.W_O" } ], "l:190": [ { "axis0": [ 103, 1 ], "axis1": [ 128, 64 ], "matrix": "L10.attn.W_Q" }, { "axis0": [ 207, 1 ], "axis1": [ 128, 64 ], "matrix": "L10.attn.W_Q" }, { "axis0": [ 327, 1 ], "axis1": [ 128, 64 ], "matrix": "L10.attn.W_Q" }, { "axis0": [ 330, 1 ], "axis1": [ 128, 64 ], "matrix": "L10.attn.W_Q" }, { "axis0": [ 60, 4 ], "axis1": [ 128, 64 ], "matrix": "L10.attn.W_K" }, { "axis0": [ 104, 17 ], "axis1": [ 128, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 45, 1 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 223, 1 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 236, 3 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 240, 12 ], "matrix": "L10.attn.W_O" } ], "l:191": [ { "axis0": [ 0, 1 ], "axis1": [ 30, 13 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 30, 13 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 103, 1 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 207, 1 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 327, 1 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 330, 1 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 326, 1 ], "axis1": [ 30, 13 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 30, 13 ], "axis1": [ 103, 1 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 30, 13 ], "axis1": [ 207, 1 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 30, 13 ], "axis1": [ 327, 1 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 30, 13 ], "axis1": [ 330, 1 ], "matrix": "L7.mlp.W_out" } ], "l:192": [ { "axis0": [ 0, 1 ], "axis1": [ 401, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 401, 100 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 419, 34 ], "matrix": "L11.mlp.W_out" }, { "axis0": [ 42, 4 ], "axis1": [ 401, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 49, 2 ], "axis1": [ 401, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 213, 9 ], "axis1": [ 401, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 223, 1 ], "axis1": [ 401, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 236, 11 ], "axis1": [ 401, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 248, 3 ], "axis1": [ 401, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 252, 4 ], "axis1": [ 401, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 401, 100 ], "axis1": [ 419, 34 ], "matrix": "L11.mlp.W_out" } ], "l:194": [ { "axis0": [ 260, 2 ], "axis1": [ 192, 64 ], "matrix": "L11.attn.W_Q" }, { "axis0": [ 21, 1 ], "axis1": [ 192, 64 ], "matrix": "L11.attn.W_K" }, { "axis0": [ 262, 1 ], "axis1": [ 192, 64 ], "matrix": "L11.attn.W_K" }, { "axis0": [ 42, 2 ], "axis1": [ 192, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 185, 5 ], "axis1": [ 192, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 210, 1 ], "axis1": [ 192, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 213, 9 ], "axis1": [ 192, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 192, 64 ], "axis1": [ 42, 4 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 213, 9 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 223, 1 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 236, 3 ], "matrix": "L11.attn.W_O" } ], "l:196": [ { "axis0": [ 0, 1 ], "axis1": [ 260, 1 ], "matrix": "L10.mlp.W_out" } ], "l:197": [ { "axis0": [ 0, 1 ], "axis1": [ 261, 1 ], "matrix": "L10.mlp.W_out" } ], "l:199": [ { "axis0": [ 0, 1 ], "axis1": [ 262, 1 ], "matrix": "L10.mlp.W_out" } ], "l:2": [ { "axis0": [ 39, 2 ], "axis1": [ 128, 64 ], "matrix": "L1.attn.W_Q" }, { "axis0": [ 21, 1 ], "axis1": [ 128, 64 ], "matrix": "L1.attn.W_K" }, { "axis0": [ 41, 1 ], "axis1": [ 128, 64 ], "matrix": "L1.attn.W_K" }, { "axis0": [ 21, 1 ], "axis1": [ 128, 64 ], "matrix": "L1.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 26, 1 ], "matrix": "L1.attn.W_O" } ], "l:20": [ { "axis0": [ 0, 1 ], "axis1": [ 48, 1 ], "matrix": "L1.mlp.W_out" } ], "l:200": [ { "axis0": [ 18, 3 ], "axis1": [ 128, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 68, 1 ], "axis1": [ 128, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 42, 4 ], "axis1": [ 128, 64 ], "matrix": "L8.attn.W_K" }, { "axis0": [ 24, 1 ], "axis1": [ 128, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 173, 16 ], "axis1": [ 128, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 42, 2 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 185, 5 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 210, 1 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 213, 9 ], "matrix": "L8.attn.W_O" } ], "l:201": [ { "axis0": [ 0, 1 ], "axis1": [ 38, 13 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 38, 13 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 18, 3 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 68, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 32, 1 ], "axis1": [ 38, 13 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 38, 13 ], "axis1": [ 18, 3 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 38, 13 ], "axis1": [ 68, 1 ], "matrix": "L4.mlp.W_out" } ], "l:202": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L19.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L19.attn.W_K" }, { "axis0": [ 299, 3 ], "axis1": [ 0, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 65, 1 ], "matrix": "L19.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 326, 1 ], "matrix": "L19.attn.W_O" } ], "l:203": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 9 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 9 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 299, 3 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 44, 1 ], "axis1": [ 1, 9 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 188, 1 ], "axis1": [ 1, 9 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 220, 1 ], "axis1": [ 1, 9 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 222, 1 ], "axis1": [ 1, 9 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 327, 1 ], "axis1": [ 1, 9 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 333, 1 ], "axis1": [ 1, 9 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 1, 9 ], "axis1": [ 299, 3 ], "matrix": "L18.mlp.W_out" } ], "l:205": [ { "axis0": [ 0, 1 ], "axis1": [ 301, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 301, 100 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 44, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 220, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 222, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 177, 2 ], "axis1": [ 301, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 180, 9 ], "axis1": [ 301, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 210, 1 ], "axis1": [ 301, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 214, 1 ], "axis1": [ 301, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 217, 1 ], "axis1": [ 301, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 333, 1 ], "axis1": [ 301, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 392, 1 ], "axis1": [ 301, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 404, 1 ], "axis1": [ 301, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 843, 17 ], "axis1": [ 301, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 301, 100 ], "axis1": [ 44, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 301, 100 ], "axis1": [ 220, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 301, 100 ], "axis1": [ 222, 1 ], "matrix": "L15.mlp.W_out" } ], "l:207": [ { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L15.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L15.attn.W_K" }, { "axis0": [ 743, 64 ], "axis1": [ 128, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 177, 2 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 180, 9 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 210, 1 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 214, 1 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 217, 1 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 333, 1 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 392, 1 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 404, 1 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L15.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L15.attn.W_K" }, { "axis0": [ 807, 36 ], "axis1": [ 192, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 192, 64 ], "axis1": [ 177, 2 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 180, 9 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 210, 1 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 214, 1 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 217, 1 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 333, 1 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 392, 1 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 404, 1 ], "matrix": "L15.attn.W_O" } ], "l:208": [ { "axis0": [ 0, 1 ], "axis1": [ 1570, 200 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 57, 1 ], "axis1": [ 1570, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 211, 1 ], "axis1": [ 1570, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 222, 1 ], "axis1": [ 1570, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 263, 1 ], "axis1": [ 1570, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 273, 12 ], "axis1": [ 1570, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 288, 16 ], "axis1": [ 1570, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 305, 2 ], "axis1": [ 1570, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 308, 11 ], "axis1": [ 1570, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 320, 3 ], "axis1": [ 1570, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 331, 11 ], "axis1": [ 1570, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 343, 2 ], "axis1": [ 1570, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 347, 1 ], "axis1": [ 1570, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 350, 1 ], "axis1": [ 1570, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 362, 27 ], "axis1": [ 1570, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 390, 9 ], "axis1": [ 1570, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 404, 1 ], "axis1": [ 1570, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 57, 1 ], "axis1": [ 1670, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 211, 1 ], "axis1": [ 1670, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 222, 1 ], "axis1": [ 1670, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 263, 1 ], "axis1": [ 1670, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 273, 12 ], "axis1": [ 1670, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 288, 16 ], "axis1": [ 1670, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 305, 2 ], "axis1": [ 1670, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 308, 11 ], "axis1": [ 1670, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 320, 3 ], "axis1": [ 1670, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 331, 11 ], "axis1": [ 1670, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 343, 2 ], "axis1": [ 1670, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 347, 1 ], "axis1": [ 1670, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 350, 1 ], "axis1": [ 1670, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 362, 27 ], "axis1": [ 1670, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 390, 9 ], "axis1": [ 1670, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 404, 1 ], "axis1": [ 1670, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1570, 100 ], "axis1": [ 743, 100 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1670, 100 ], "axis1": [ 743, 100 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 743, 100 ], "matrix": "L14.mlp.W_out" } ], "l:209": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 400 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 400 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 57, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 211, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 222, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 263, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 273, 12 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 288, 16 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 305, 2 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 308, 11 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 320, 3 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 331, 11 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 343, 2 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 347, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 350, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 362, 27 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 390, 9 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 404, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 120, 1 ], "axis1": [ 1, 400 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 355, 1 ], "axis1": [ 1, 400 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 1, 400 ], "axis1": [ 57, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 1, 400 ], "axis1": [ 211, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 1, 400 ], "axis1": [ 222, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 1, 400 ], "axis1": [ 263, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 1, 400 ], "axis1": [ 273, 12 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 1, 400 ], "axis1": [ 288, 16 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 1, 400 ], "axis1": [ 305, 2 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 1, 400 ], "axis1": [ 308, 11 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 1, 400 ], "axis1": [ 320, 3 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 1, 400 ], "axis1": [ 331, 11 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 1, 400 ], "axis1": [ 343, 2 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 1, 400 ], "axis1": [ 347, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 1, 400 ], "axis1": [ 350, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 1, 400 ], "axis1": [ 362, 27 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 1, 400 ], "axis1": [ 390, 9 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 1, 400 ], "axis1": [ 404, 1 ], "matrix": "L13.mlp.W_out" } ], "l:21": [ { "axis0": [ 0, 1 ], "axis1": [ 17, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 21, 1 ], "axis1": [ 17, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 21, 1 ], "axis1": [ 18, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 17, 1 ], "axis1": [ 47, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 18, 1 ], "axis1": [ 47, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 47, 1 ], "matrix": "L1.mlp.W_out" } ], "l:211": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L12.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L12.attn.W_K" }, { "axis0": [ 57, 2 ], "axis1": [ 0, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 273, 12 ], "axis1": [ 0, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 288, 20 ], "axis1": [ 0, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 120, 1 ], "matrix": "L12.attn.W_O" } ], "l:212": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 100 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 57, 2 ], "matrix": "L11.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 273, 12 ], "matrix": "L11.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 288, 20 ], "matrix": "L11.mlp.W_out" }, { "axis0": [ 18, 8 ], "axis1": [ 1, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 55, 2 ], "axis1": [ 1, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 59, 8 ], "axis1": [ 1, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 68, 1 ], "axis1": [ 1, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 102, 9 ], "axis1": [ 1, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 137, 2 ], "axis1": [ 1, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 173, 1 ], "axis1": [ 1, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 326, 1 ], "axis1": [ 1, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 328, 2 ], "axis1": [ 1, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 1, 100 ], "axis1": [ 57, 2 ], "matrix": "L11.mlp.W_out" }, { "axis0": [ 1, 100 ], "axis1": [ 273, 12 ], "matrix": "L11.mlp.W_out" }, { "axis0": [ 1, 100 ], "axis1": [ 288, 20 ], "matrix": "L11.mlp.W_out" } ], "l:214": [ { "axis0": [ 252, 2 ], "axis1": [ 0, 64 ], "matrix": "L11.attn.W_Q" }, { "axis0": [ 21, 1 ], "axis1": [ 0, 64 ], "matrix": "L11.attn.W_K" }, { "axis0": [ 254, 1 ], "axis1": [ 0, 64 ], "matrix": "L11.attn.W_K" }, { "axis0": [ 18, 3 ], "axis1": [ 0, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 22, 2 ], "axis1": [ 0, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 44, 1 ], "axis1": [ 0, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 50, 5 ], "axis1": [ 0, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 64, 2 ], "axis1": [ 0, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 67, 2 ], "axis1": [ 0, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 102, 1 ], "axis1": [ 0, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 326, 1 ], "axis1": [ 0, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 18, 3 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 55, 2 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 59, 7 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 68, 1 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 106, 1 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 326, 1 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 328, 2 ], "matrix": "L11.attn.W_O" } ], "l:216": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L10.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 252, 1 ], "matrix": "L10.mlp.W_out" } ], "l:217": [ { "axis0": [ 0, 1 ], "axis1": [ 253, 1 ], "matrix": "L10.mlp.W_out" } ], "l:219": [ { "axis0": [ 0, 1 ], "axis1": [ 254, 1 ], "matrix": "L10.mlp.W_out" } ], "l:22": [ { "axis0": [ 0, 1 ], "axis1": [ 5, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 5, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 5, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 29, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 24, 1 ], "axis1": [ 5, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 5, 2 ], "axis1": [ 29, 1 ], "matrix": "L3.mlp.W_out" } ], "l:220": [ { "axis0": [ 64, 4 ], "axis1": [ 0, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 42, 4 ], "axis1": [ 0, 64 ], "matrix": "L8.attn.W_K" }, { "axis0": [ 24, 1 ], "axis1": [ 0, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 173, 16 ], "axis1": [ 0, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 18, 3 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 22, 2 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 44, 1 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 50, 5 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 64, 2 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 67, 2 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 102, 1 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 326, 1 ], "matrix": "L8.attn.W_O" } ], "l:221": [ { "axis0": [ 0, 1 ], "axis1": [ 21, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 21, 17 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 4 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 32, 1 ], "axis1": [ 21, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 21, 17 ], "axis1": [ 64, 4 ], "matrix": "L4.mlp.W_out" } ], "l:222": [ { "axis0": [ 255, 2 ], "axis1": [ 64, 64 ], "matrix": "L11.attn.W_Q" }, { "axis0": [ 21, 1 ], "axis1": [ 64, 64 ], "matrix": "L11.attn.W_K" }, { "axis0": [ 257, 1 ], "axis1": [ 64, 64 ], "matrix": "L11.attn.W_K" }, { "axis0": [ 55, 2 ], "axis1": [ 64, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 59, 5 ], "axis1": [ 64, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 103, 8 ], "axis1": [ 64, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 328, 2 ], "axis1": [ 64, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 21, 5 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 66, 1 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 102, 4 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 107, 4 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 137, 2 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 173, 1 ], "matrix": "L11.attn.W_O" } ], "l:224": [ { "axis0": [ 0, 1 ], "axis1": [ 255, 1 ], "matrix": "L10.mlp.W_out" } ], "l:225": [ { "axis0": [ 0, 1 ], "axis1": [ 256, 1 ], "matrix": "L10.mlp.W_out" } ], "l:227": [ { "axis0": [ 0, 1 ], "axis1": [ 257, 1 ], "matrix": "L10.mlp.W_out" } ], "l:228": [ { "axis0": [ 55, 2 ], "axis1": [ 0, 64 ], "matrix": "L10.attn.W_Q" }, { "axis0": [ 328, 2 ], "axis1": [ 0, 64 ], "matrix": "L10.attn.W_Q" }, { "axis0": [ 60, 4 ], "axis1": [ 0, 64 ], "matrix": "L10.attn.W_K" }, { "axis0": [ 104, 17 ], "axis1": [ 0, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 55, 2 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 59, 5 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 103, 8 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 328, 2 ], "matrix": "L10.attn.W_O" } ], "l:229": [ { "axis0": [ 0, 1 ], "axis1": [ 13, 17 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 13, 17 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 55, 2 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 328, 2 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 326, 1 ], "axis1": [ 13, 17 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 13, 17 ], "axis1": [ 55, 2 ], "matrix": "L7.mlp.W_out" }, { "axis0": [ 13, 17 ], "axis1": [ 328, 2 ], "matrix": "L7.mlp.W_out" } ], "l:23": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L27.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L27.attn.W_K" }, { "axis0": [ 39, 1 ], "axis1": [ 0, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 45, 2 ], "axis1": [ 0, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 49, 8 ], "axis1": [ 0, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 58, 7 ], "axis1": [ 0, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 67, 2 ], "axis1": [ 0, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 121, 1 ], "axis1": [ 0, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 152, 4 ], "axis1": [ 0, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 194, 3 ], "axis1": [ 0, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 223, 4 ], "axis1": [ 0, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 228, 3 ], "axis1": [ 0, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 236, 11 ], "axis1": [ 0, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 248, 3 ], "axis1": [ 0, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 252, 2 ], "axis1": [ 0, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 255, 2 ], "axis1": [ 0, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 258, 6 ], "axis1": [ 0, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 265, 2 ], "axis1": [ 0, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 268, 3 ], "axis1": [ 0, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 18, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 65, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 81, 13 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 326, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 329, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L27.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L27.attn.W_K" }, { "axis0": [ 273, 1 ], "axis1": [ 64, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 290, 12 ], "axis1": [ 64, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 303, 1 ], "axis1": [ 64, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 305, 2 ], "axis1": [ 64, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 308, 4 ], "axis1": [ 64, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 313, 1 ], "axis1": [ 64, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 315, 2 ], "axis1": [ 64, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 318, 1 ], "axis1": [ 64, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 320, 3 ], "axis1": [ 64, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 328, 1 ], "axis1": [ 64, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 335, 2 ], "axis1": [ 64, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 338, 1 ], "axis1": [ 64, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 340, 5 ], "axis1": [ 64, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 351, 1 ], "axis1": [ 64, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 354, 1 ], "axis1": [ 64, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 356, 26 ], "axis1": [ 64, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 18, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 65, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 81, 13 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 326, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 329, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L27.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L27.attn.W_K" }, { "axis0": [ 382, 1 ], "axis1": [ 128, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 384, 5 ], "axis1": [ 128, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 390, 2 ], "axis1": [ 128, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 393, 2 ], "axis1": [ 128, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 398, 6 ], "axis1": [ 128, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 405, 9 ], "axis1": [ 128, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 415, 39 ], "axis1": [ 128, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 18, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 65, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 81, 13 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 326, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 329, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L27.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L27.attn.W_K" }, { "axis0": [ 454, 12 ], "axis1": [ 192, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 192, 64 ], "axis1": [ 18, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 65, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 81, 13 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 326, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 329, 1 ], "matrix": "L27.attn.W_O" } ], "l:230": [ { "axis0": [ 0, 1 ], "axis1": [ 31, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 120, 1 ], "axis1": [ 31, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 120, 1 ], "axis1": [ 32, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 31, 1 ], "axis1": [ 355, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 32, 1 ], "axis1": [ 355, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 355, 1 ], "matrix": "L12.mlp.W_out" } ], "l:231": [ { "axis0": [ 0, 1 ], "axis1": [ 843, 17 ], "matrix": "L14.mlp.W_out" } ], "l:232": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 9 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 9 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 188, 1 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 327, 1 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 333, 1 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 340, 1 ], "axis1": [ 1, 9 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 362, 2 ], "axis1": [ 1, 9 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 365, 3 ], "axis1": [ 1, 9 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 1, 9 ], "axis1": [ 188, 1 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 1, 9 ], "axis1": [ 327, 1 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 1, 9 ], "axis1": [ 333, 1 ], "matrix": "L17.mlp.W_out" } ], "l:234": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 9 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 9 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 340, 1 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 362, 2 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 42, 1 ], "axis1": [ 1, 9 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 211, 1 ], "axis1": [ 1, 9 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 213, 1 ], "axis1": [ 1, 9 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 215, 2 ], "axis1": [ 1, 9 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 334, 1 ], "axis1": [ 1, 9 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 1, 9 ], "axis1": [ 340, 1 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 1, 9 ], "axis1": [ 362, 2 ], "matrix": "L16.mlp.W_out" } ], "l:236": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 100 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 42, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 211, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 213, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 52, 3 ], "axis1": [ 1, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 56, 1 ], "axis1": [ 1, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 58, 6 ], "axis1": [ 1, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 65, 1 ], "axis1": [ 1, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 173, 4 ], "axis1": [ 1, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 179, 1 ], "axis1": [ 1, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 207, 1 ], "axis1": [ 1, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 209, 1 ], "axis1": [ 1, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 212, 1 ], "axis1": [ 1, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 256, 1 ], "axis1": [ 1, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 264, 2 ], "axis1": [ 1, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 326, 1 ], "axis1": [ 1, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 351, 5 ], "axis1": [ 1, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 357, 5 ], "axis1": [ 1, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 542, 1 ], "axis1": [ 1, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1, 100 ], "axis1": [ 42, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 1, 100 ], "axis1": [ 211, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 1, 100 ], "axis1": [ 213, 1 ], "matrix": "L15.mlp.W_out" } ], "l:238": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 402 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 402 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 207, 1 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 209, 1 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 212, 1 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 256, 1 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 264, 2 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 351, 5 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 357, 5 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 542, 1 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 356, 1 ], "axis1": [ 1, 402 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1, 402 ], "axis1": [ 207, 1 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 1, 402 ], "axis1": [ 209, 1 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 1, 402 ], "axis1": [ 212, 1 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 1, 402 ], "axis1": [ 256, 1 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 1, 402 ], "axis1": [ 264, 2 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 1, 402 ], "axis1": [ 351, 5 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 1, 402 ], "axis1": [ 357, 5 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 1, 402 ], "axis1": [ 542, 1 ], "matrix": "L14.mlp.W_out" } ], "l:239": [ { "axis0": [ 0, 1 ], "axis1": [ 33, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 57, 2 ], "axis1": [ 33, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 273, 12 ], "axis1": [ 33, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 288, 20 ], "axis1": [ 33, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 487, 34 ], "axis1": [ 33, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 555, 34 ], "axis1": [ 33, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 57, 2 ], "axis1": [ 34, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 273, 12 ], "axis1": [ 34, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 288, 20 ], "axis1": [ 34, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 487, 34 ], "axis1": [ 34, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 555, 34 ], "axis1": [ 34, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 33, 1 ], "axis1": [ 356, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 34, 1 ], "axis1": [ 356, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" } ], "l:24": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 18, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 28, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 28, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 28, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 28, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 28, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 28, 11 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 40, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 42, 3 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 57, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 65, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 122, 13 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 134, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 134, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 134, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 134, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 134, 2 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 135, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 135, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 135, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 135, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 135, 2 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 136, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 136, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 136, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 136, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 136, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 139, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 139, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 139, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 139, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 139, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 139, 2 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 140, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 140, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 140, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 140, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 140, 2 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 141, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 141, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 141, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 141, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 141, 2 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 142, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 142, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 142, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 142, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 142, 2 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 143, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 143, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 143, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 143, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 143, 2 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 144, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 144, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 144, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 144, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 144, 2 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 145, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 145, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 145, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 145, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 145, 2 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 146, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 146, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 146, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 146, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 146, 2 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 147, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 147, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 147, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 147, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 147, 2 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 148, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 148, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 148, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 148, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 148, 2 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 149, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 149, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 149, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 149, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 149, 2 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 150, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 150, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 150, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 150, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 150, 2 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 151, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 151, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 151, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 151, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 151, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 156, 33 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 190, 4 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 197, 24 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 222, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 274, 11 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 288, 2 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 302, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 326, 2 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 329, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 333, 2 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 337, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 339, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 392, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 404, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 18, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 28, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 28, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 28, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 28, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 28, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 28, 11 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 40, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 42, 3 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 57, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 65, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 122, 13 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 134, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 134, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 134, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 134, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 134, 2 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 135, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 135, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 135, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 135, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 135, 2 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 136, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 136, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 136, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 136, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 136, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 139, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 139, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 139, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 139, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 139, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 139, 2 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 140, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 140, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 140, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 140, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 140, 2 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 141, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 141, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 141, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 141, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 141, 2 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 142, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 142, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 142, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 142, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 142, 2 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 143, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 143, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 143, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 143, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 143, 2 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 144, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 144, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 144, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 144, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 144, 2 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 145, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 145, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 145, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 145, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 145, 2 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 146, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 146, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 146, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 146, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 146, 2 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 147, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 147, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 147, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 147, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 147, 2 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 148, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 148, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 148, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 148, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 148, 2 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 149, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 149, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 149, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 149, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 149, 2 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 150, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 150, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 150, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 150, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 150, 2 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 151, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 151, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 151, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 151, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 151, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 156, 33 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 190, 4 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 197, 24 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 222, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 274, 11 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 288, 2 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 302, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 326, 2 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 329, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 333, 2 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 337, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 339, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 392, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 404, 1 ], "axis1": [ 1, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 1, 204 ], "axis1": [ 39, 1 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 45, 2 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 49, 8 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 58, 7 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 67, 2 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 121, 1 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 152, 4 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 194, 3 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 223, 4 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 228, 3 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 236, 11 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 248, 3 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 252, 2 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 255, 2 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 258, 6 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 265, 2 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 268, 3 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 273, 1 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 290, 12 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 303, 1 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 305, 2 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 308, 4 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 313, 1 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 315, 2 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 318, 1 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 320, 3 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 328, 1 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 335, 2 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 338, 1 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 340, 5 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 351, 1 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 354, 1 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 356, 27 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 384, 5 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 390, 2 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 393, 2 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 398, 6 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 405, 9 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 1, 204 ], "axis1": [ 415, 51 ], "matrix": "L26.mlp.W_out" } ], "l:241": [ { "axis0": [ 0, 1 ], "axis1": [ 601, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 601, 100 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 487, 34 ], "matrix": "L11.mlp.W_out" }, { "axis0": [ 21, 5 ], "axis1": [ 601, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 42, 4 ], "axis1": [ 601, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 66, 1 ], "axis1": [ 601, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 102, 4 ], "axis1": [ 601, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 107, 4 ], "axis1": [ 601, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 137, 2 ], "axis1": [ 601, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 173, 1 ], "axis1": [ 601, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 213, 9 ], "axis1": [ 601, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 223, 1 ], "axis1": [ 601, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 236, 3 ], "axis1": [ 601, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 601, 100 ], "axis1": [ 487, 34 ], "matrix": "L11.mlp.W_out" } ], "l:243": [ { "axis0": [ 0, 1 ], "axis1": [ 801, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 801, 100 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 555, 34 ], "matrix": "L11.mlp.W_out" }, { "axis0": [ 18, 3 ], "axis1": [ 801, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 51, 6 ], "axis1": [ 801, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 59, 7 ], "axis1": [ 801, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 67, 2 ], "axis1": [ 801, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 106, 1 ], "axis1": [ 801, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 258, 5 ], "axis1": [ 801, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 266, 7 ], "axis1": [ 801, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 326, 1 ], "axis1": [ 801, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 328, 2 ], "axis1": [ 801, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 801, 100 ], "axis1": [ 555, 34 ], "matrix": "L11.mlp.W_out" } ], "l:245": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L15.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L15.attn.W_K" }, { "axis0": [ 743, 64 ], "axis1": [ 0, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 52, 3 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 56, 1 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 58, 6 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 65, 1 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 173, 4 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 179, 1 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 326, 1 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L15.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L15.attn.W_K" }, { "axis0": [ 807, 36 ], "axis1": [ 64, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 52, 3 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 56, 1 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 58, 6 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 65, 1 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 173, 4 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 179, 1 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 326, 1 ], "matrix": "L15.attn.W_O" } ], "l:246": [ { "axis0": [ 0, 1 ], "axis1": [ 101, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 101, 100 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 215, 2 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 334, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 111, 1 ], "axis1": [ 101, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 342, 1 ], "axis1": [ 101, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 399, 5 ], "axis1": [ 101, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 405, 9 ], "axis1": [ 101, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 415, 2 ], "axis1": [ 101, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 543, 16 ], "axis1": [ 101, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 101, 100 ], "axis1": [ 215, 2 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 101, 100 ], "axis1": [ 334, 1 ], "matrix": "L15.mlp.W_out" } ], "l:248": [ { "axis0": [ 0, 1 ], "axis1": [ 401, 402 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 401, 402 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 342, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 399, 5 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 405, 9 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 415, 2 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 327, 1 ], "axis1": [ 401, 402 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 401, 402 ], "axis1": [ 342, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 401, 402 ], "axis1": [ 399, 5 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 401, 402 ], "axis1": [ 405, 9 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 401, 402 ], "axis1": [ 415, 2 ], "matrix": "L13.mlp.W_out" } ], "l:249": [ { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L12.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L12.attn.W_K" }, { "axis0": [ 385, 34 ], "axis1": [ 64, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 453, 30 ], "axis1": [ 64, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 327, 1 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L12.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L12.attn.W_K" }, { "axis0": [ 483, 64 ], "axis1": [ 128, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 327, 1 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L12.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L12.attn.W_K" }, { "axis0": [ 547, 42 ], "axis1": [ 192, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 192, 64 ], "axis1": [ 327, 1 ], "matrix": "L12.attn.W_O" } ], "l:251": [ { "axis0": [ 0, 1 ], "axis1": [ 301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 301, 100 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 385, 34 ], "matrix": "L11.mlp.W_out" }, { "axis0": [ 21, 5 ], "axis1": [ 301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 66, 1 ], "axis1": [ 301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 102, 4 ], "axis1": [ 301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 107, 4 ], "axis1": [ 301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 137, 2 ], "axis1": [ 301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 173, 17 ], "axis1": [ 301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 210, 1 ], "axis1": [ 301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 301, 100 ], "axis1": [ 385, 34 ], "matrix": "L11.mlp.W_out" } ], "l:253": [ { "axis0": [ 0, 1 ], "axis1": [ 501, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 501, 100 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 453, 34 ], "matrix": "L11.mlp.W_out" }, { "axis0": [ 42, 4 ], "axis1": [ 501, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 51, 4 ], "axis1": [ 501, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 67, 1 ], "axis1": [ 501, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 213, 9 ], "axis1": [ 501, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 223, 1 ], "axis1": [ 501, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 236, 3 ], "axis1": [ 501, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 258, 5 ], "axis1": [ 501, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 266, 7 ], "axis1": [ 501, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 501, 100 ], "axis1": [ 453, 34 ], "matrix": "L11.mlp.W_out" } ], "l:255": [ { "axis0": [ 0, 1 ], "axis1": [ 701, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 701, 100 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 521, 34 ], "matrix": "L11.mlp.W_out" }, { "axis0": [ 18, 3 ], "axis1": [ 701, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 49, 2 ], "axis1": [ 701, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 55, 2 ], "axis1": [ 701, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 59, 7 ], "axis1": [ 701, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 68, 1 ], "axis1": [ 701, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 106, 1 ], "axis1": [ 701, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 239, 8 ], "axis1": [ 701, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 248, 3 ], "axis1": [ 701, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 252, 4 ], "axis1": [ 701, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 326, 1 ], "axis1": [ 701, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 328, 2 ], "axis1": [ 701, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 701, 100 ], "axis1": [ 521, 34 ], "matrix": "L11.mlp.W_out" } ], "l:257": [ { "axis0": [ 0, 1 ], "axis1": [ 403, 42 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 403, 42 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 111, 1 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 543, 16 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 356, 1 ], "axis1": [ 403, 42 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 403, 42 ], "axis1": [ 111, 1 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 403, 42 ], "axis1": [ 543, 16 ], "matrix": "L14.mlp.W_out" } ], "l:258": [ { "axis0": [ 0, 1 ], "axis1": [ 10, 9 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 10, 9 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 365, 3 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 43, 1 ], "axis1": [ 10, 9 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 115, 1 ], "axis1": [ 10, 9 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 218, 2 ], "axis1": [ 10, 9 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 577, 2 ], "axis1": [ 10, 9 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 10, 9 ], "axis1": [ 365, 3 ], "matrix": "L16.mlp.W_out" } ], "l:26": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 46 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 46 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L25.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 122, 12 ], "matrix": "L25.mlp.W_out" }, { "axis0": [ 121, 1 ], "axis1": [ 1, 46 ], "matrix": "L25.mlp.W_in" }, { "axis0": [ 1, 46 ], "axis1": [ 122, 12 ], "matrix": "L25.mlp.W_out" } ], "l:260": [ { "axis0": [ 0, 1 ], "axis1": [ 847, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 847, 100 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 115, 1 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 577, 2 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 417, 34 ], "axis1": [ 847, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 847, 100 ], "axis1": [ 115, 1 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 847, 100 ], "axis1": [ 577, 2 ], "matrix": "L14.mlp.W_out" } ], "l:262": [ { "axis0": [ 0, 1 ], "axis1": [ 845, 402 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 845, 402 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 434, 17 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 207, 1 ], "axis1": [ 845, 402 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 845, 402 ], "axis1": [ 434, 17 ], "matrix": "L13.mlp.W_out" } ], "l:263": [ { "axis0": [ 0, 1 ], "axis1": [ 256, 64 ], "matrix": "L12.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 256, 64 ], "matrix": "L12.attn.W_K" }, { "axis0": [ 351, 64 ], "axis1": [ 256, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 256, 64 ], "axis1": [ 207, 1 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 320, 64 ], "matrix": "L12.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 320, 64 ], "matrix": "L12.attn.W_K" }, { "axis0": [ 415, 64 ], "axis1": [ 320, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 320, 64 ], "axis1": [ 207, 1 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 384, 64 ], "matrix": "L12.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 384, 64 ], "matrix": "L12.attn.W_K" }, { "axis0": [ 479, 8 ], "axis1": [ 384, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 521, 34 ], "axis1": [ 384, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 384, 64 ], "axis1": [ 207, 1 ], "matrix": "L12.attn.W_O" } ], "l:265": [ { "axis0": [ 0, 1 ], "axis1": [ 803, 42 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 803, 42 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 417, 17 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 327, 1 ], "axis1": [ 803, 42 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 803, 42 ], "axis1": [ 417, 17 ], "matrix": "L13.mlp.W_out" } ], "l:266": [ { "axis0": [ 0, 1 ], "axis1": [ 201, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 201, 100 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 43, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 218, 2 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 112, 3 ], "axis1": [ 201, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 348, 1 ], "axis1": [ 201, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 451, 17 ], "axis1": [ 201, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 560, 3 ], "axis1": [ 201, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 565, 4 ], "axis1": [ 201, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 570, 4 ], "axis1": [ 201, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 575, 2 ], "axis1": [ 201, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 201, 100 ], "axis1": [ 43, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 201, 100 ], "axis1": [ 218, 2 ], "matrix": "L15.mlp.W_out" } ], "l:268": [ { "axis0": [ 0, 1 ], "axis1": [ 445, 402 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 445, 402 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 112, 3 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 348, 1 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 560, 3 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 565, 4 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 570, 4 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 575, 2 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 208, 1 ], "axis1": [ 445, 402 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 445, 402 ], "axis1": [ 112, 3 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 445, 402 ], "axis1": [ 348, 1 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 445, 402 ], "axis1": [ 560, 3 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 445, 402 ], "axis1": [ 565, 4 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 445, 402 ], "axis1": [ 570, 4 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 445, 402 ], "axis1": [ 575, 2 ], "matrix": "L14.mlp.W_out" } ], "l:269": [ { "axis0": [ 0, 1 ], "axis1": [ 1247, 42 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1247, 42 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 451, 17 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 207, 1 ], "axis1": [ 1247, 42 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 1247, 42 ], "axis1": [ 451, 17 ], "matrix": "L13.mlp.W_out" } ], "l:27": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 2 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 364, 1 ], "axis1": [ 1, 1 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 364, 1 ], "axis1": [ 2, 1 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 1, 1 ], "axis1": [ 121, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 2, 1 ], "axis1": [ 121, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" } ], "l:270": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 2 ], "matrix": "L22.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 2 ], "matrix": "L22.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 2 ], "matrix": "L22.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L22.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L22.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 122, 1 ], "matrix": "L22.mlp.W_out" }, { "axis0": [ 121, 1 ], "axis1": [ 1, 2 ], "matrix": "L22.mlp.W_in" }, { "axis0": [ 225, 1 ], "axis1": [ 1, 2 ], "matrix": "L22.mlp.W_in" }, { "axis0": [ 1, 2 ], "axis1": [ 122, 1 ], "matrix": "L22.mlp.W_out" } ], "l:272": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L22.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L22.attn.W_K" }, { "axis0": [ 223, 1 ], "axis1": [ 0, 64 ], "matrix": "L22.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 121, 1 ], "matrix": "L22.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 121, 1 ], "matrix": "L22.mlp.W_out" } ], "l:273": [ { "axis0": [ 0, 1 ], "axis1": [ 3, 2 ], "matrix": "L21.mlp.W_up" }, { "axis0": [ 50, 1 ], "axis1": [ 3, 1 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 50, 1 ], "axis1": [ 4, 1 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 3, 1 ], "axis1": [ 225, 1 ], "diagonal": true, "matrix": "L21.mlp.W_out" }, { "axis0": [ 4, 1 ], "axis1": [ 225, 1 ], "diagonal": true, "matrix": "L21.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 225, 1 ], "matrix": "L21.mlp.W_out" } ], "l:274": [ { "axis0": [ 0, 1 ], "axis1": [ 18, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 18, 1 ], "matrix": "L20.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L20.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 50, 1 ], "matrix": "L20.mlp.W_out" }, { "axis0": [ 18, 1 ], "axis1": [ 18, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 29, 4 ], "axis1": [ 18, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 34, 1 ], "axis1": [ 18, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 65, 1 ], "axis1": [ 18, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 156, 8 ], "axis1": [ 18, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 326, 1 ], "axis1": [ 18, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 329, 1 ], "axis1": [ 18, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 18, 1 ], "axis1": [ 50, 1 ], "matrix": "L20.mlp.W_out" } ], "l:275": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L20.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L20.attn.W_K" }, { "axis0": [ 29, 4 ], "axis1": [ 0, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 34, 1 ], "axis1": [ 0, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 36, 4 ], "axis1": [ 0, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 52, 4 ], "axis1": [ 0, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 67, 1 ], "axis1": [ 0, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 156, 17 ], "axis1": [ 0, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 190, 8 ], "axis1": [ 0, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 258, 4 ], "axis1": [ 0, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 264, 3 ], "axis1": [ 0, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 268, 3 ], "axis1": [ 0, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 273, 1 ], "axis1": [ 0, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 391, 1 ], "axis1": [ 0, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 18, 1 ], "matrix": "L20.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 29, 4 ], "matrix": "L20.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 34, 1 ], "matrix": "L20.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 65, 1 ], "matrix": "L20.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 156, 8 ], "matrix": "L20.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 326, 1 ], "matrix": "L20.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 329, 1 ], "matrix": "L20.attn.W_O" } ], "l:277": [ { "axis0": [ 0, 1 ], "axis1": [ 77, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 77, 17 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 33, 1 ], "axis1": [ 77, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 69, 17 ], "axis1": [ 77, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 33, 1 ], "axis1": [ 77, 17 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 69, 17 ], "axis1": [ 77, 17 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 77, 17 ], "axis1": [ 156, 17 ], "matrix": "L4.mlp.W_out" } ], "l:279": [ { "axis0": [ 0, 1 ], "axis1": [ 86, 17 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 86, 17 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 35, 1 ], "axis1": [ 86, 17 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 306, 17 ], "axis1": [ 86, 17 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 35, 1 ], "axis1": [ 86, 17 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 306, 17 ], "axis1": [ 86, 17 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 86, 17 ], "axis1": [ 29, 4 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 86, 17 ], "axis1": [ 34, 1 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 86, 17 ], "axis1": [ 36, 4 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 86, 17 ], "axis1": [ 190, 8 ], "matrix": "L5.mlp.W_out" } ], "l:28": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L24.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L24.attn.W_K" }, { "axis0": [ 121, 4 ], "axis1": [ 0, 64 ], "matrix": "L24.attn.W_V" }, { "axis0": [ 127, 1 ], "axis1": [ 0, 64 ], "matrix": "L24.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 121, 1 ], "matrix": "L24.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 121, 1 ], "matrix": "L24.mlp.W_out" } ], "l:281": [ { "axis0": [ 0, 1 ], "axis1": [ 201, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 201, 17 ], "matrix": "L19.mlp.W_up" }, { "axis0": [ 40, 1 ], "axis1": [ 201, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 365, 12 ], "axis1": [ 201, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 399, 5 ], "axis1": [ 201, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 40, 1 ], "axis1": [ 201, 17 ], "matrix": "L19.mlp.W_up" }, { "axis0": [ 365, 12 ], "axis1": [ 201, 17 ], "matrix": "L19.mlp.W_up" }, { "axis0": [ 399, 5 ], "axis1": [ 201, 17 ], "matrix": "L19.mlp.W_up" }, { "axis0": [ 201, 17 ], "axis1": [ 52, 4 ], "matrix": "L19.mlp.W_out" }, { "axis0": [ 201, 17 ], "axis1": [ 67, 1 ], "matrix": "L19.mlp.W_out" }, { "axis0": [ 201, 17 ], "axis1": [ 258, 4 ], "matrix": "L19.mlp.W_out" }, { "axis0": [ 201, 17 ], "axis1": [ 264, 3 ], "matrix": "L19.mlp.W_out" }, { "axis0": [ 201, 17 ], "axis1": [ 268, 3 ], "matrix": "L19.mlp.W_out" }, { "axis0": [ 201, 17 ], "axis1": [ 273, 1 ], "matrix": "L19.mlp.W_out" }, { "axis0": [ 201, 17 ], "axis1": [ 391, 1 ], "matrix": "L19.mlp.W_out" } ], "l:283": [ { "axis0": [ 0, 1 ], "axis1": [ 210, 200 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 210, 200 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 365, 12 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 399, 5 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 43, 1 ], "axis1": [ 210, 200 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 112, 3 ], "axis1": [ 210, 200 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 273, 1 ], "axis1": [ 210, 200 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 348, 1 ], "axis1": [ 210, 200 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 451, 17 ], "axis1": [ 210, 200 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 560, 3 ], "axis1": [ 210, 200 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 565, 4 ], "axis1": [ 210, 200 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 570, 4 ], "axis1": [ 210, 200 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 575, 2 ], "axis1": [ 210, 200 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 210, 200 ], "axis1": [ 365, 12 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 210, 200 ], "axis1": [ 399, 5 ], "matrix": "L18.mlp.W_out" } ], "l:285": [ { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L18.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L18.attn.W_K" }, { "axis0": [ 207, 1 ], "axis1": [ 64, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 209, 1 ], "axis1": [ 64, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 211, 1 ], "axis1": [ 64, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 43, 1 ], "matrix": "L18.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 273, 1 ], "matrix": "L18.attn.W_O" } ], "l:286": [ { "axis0": [ 0, 1 ], "axis1": [ 19, 9 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 19, 9 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 207, 1 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 209, 1 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 211, 1 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 368, 6 ], "axis1": [ 19, 9 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 19, 9 ], "axis1": [ 207, 1 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 19, 9 ], "axis1": [ 209, 1 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 19, 9 ], "axis1": [ 211, 1 ], "matrix": "L17.mlp.W_out" } ], "l:288": [ { "axis0": [ 0, 1 ], "axis1": [ 19, 9 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 19, 9 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 368, 3 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 42, 1 ], "axis1": [ 19, 9 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 44, 1 ], "axis1": [ 19, 9 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 211, 1 ], "axis1": [ 19, 9 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 213, 1 ], "axis1": [ 19, 9 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 220, 1 ], "axis1": [ 19, 9 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 222, 1 ], "axis1": [ 19, 9 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 19, 9 ], "axis1": [ 368, 3 ], "matrix": "L16.mlp.W_out" } ], "l:290": [ { "axis0": [ 0, 1 ], "axis1": [ 28, 9 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 28, 9 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 371, 3 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 115, 1 ], "axis1": [ 28, 9 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 215, 2 ], "axis1": [ 28, 9 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 334, 1 ], "axis1": [ 28, 9 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 577, 2 ], "axis1": [ 28, 9 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 28, 9 ], "axis1": [ 371, 3 ], "matrix": "L16.mlp.W_out" } ], "l:292": [ { "axis0": [ 0, 1 ], "axis1": [ 3, 2 ], "matrix": "L22.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 3, 2 ], "matrix": "L22.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 3, 2 ], "matrix": "L22.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 123, 1 ], "matrix": "L22.mlp.W_out" }, { "axis0": [ 121, 1 ], "axis1": [ 3, 2 ], "matrix": "L22.mlp.W_in" }, { "axis0": [ 225, 2 ], "axis1": [ 3, 2 ], "matrix": "L22.mlp.W_in" }, { "axis0": [ 3, 2 ], "axis1": [ 123, 1 ], "matrix": "L22.mlp.W_out" } ], "l:294": [ { "axis0": [ 0, 1 ], "axis1": [ 5, 2 ], "matrix": "L21.mlp.W_up" }, { "axis0": [ 253, 1 ], "axis1": [ 5, 1 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 253, 1 ], "axis1": [ 6, 1 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 5, 1 ], "axis1": [ 226, 1 ], "diagonal": true, "matrix": "L21.mlp.W_out" }, { "axis0": [ 6, 1 ], "axis1": [ 226, 1 ], "diagonal": true, "matrix": "L21.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 226, 1 ], "matrix": "L21.mlp.W_out" } ], "l:295": [ { "axis0": [ 0, 1 ], "axis1": [ 19, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 19, 1 ], "matrix": "L20.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 253, 1 ], "matrix": "L20.mlp.W_out" }, { "axis0": [ 36, 3 ], "axis1": [ 19, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 164, 9 ], "axis1": [ 19, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 190, 4 ], "axis1": [ 19, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 197, 1 ], "axis1": [ 19, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 19, 1 ], "axis1": [ 253, 1 ], "matrix": "L20.mlp.W_out" } ], "l:296": [ { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L20.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L20.attn.W_K" }, { "axis0": [ 46, 1 ], "axis1": [ 64, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 49, 2 ], "axis1": [ 64, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 56, 1 ], "axis1": [ 64, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 58, 2 ], "axis1": [ 64, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 63, 1 ], "axis1": [ 64, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 198, 9 ], "axis1": [ 64, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 224, 7 ], "axis1": [ 64, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 240, 7 ], "axis1": [ 64, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 248, 3 ], "axis1": [ 64, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 252, 4 ], "axis1": [ 64, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 290, 9 ], "axis1": [ 64, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 316, 1 ], "axis1": [ 64, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 318, 1 ], "axis1": [ 64, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 320, 1 ], "axis1": [ 64, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 328, 1 ], "axis1": [ 64, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 335, 1 ], "axis1": [ 64, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 36, 3 ], "matrix": "L20.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 164, 9 ], "matrix": "L20.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 190, 4 ], "matrix": "L20.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 197, 1 ], "matrix": "L20.attn.W_O" } ], "l:298": [ { "axis0": [ 0, 1 ], "axis1": [ 601, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 601, 17 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 33, 1 ], "axis1": [ 601, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 708, 17 ], "axis1": [ 601, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 33, 1 ], "axis1": [ 601, 17 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 708, 17 ], "axis1": [ 601, 17 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 601, 17 ], "axis1": [ 49, 2 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 601, 17 ], "axis1": [ 240, 7 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 601, 17 ], "axis1": [ 248, 3 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 601, 17 ], "axis1": [ 252, 4 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 601, 17 ], "axis1": [ 335, 1 ], "matrix": "L15.mlp.W_out" } ], "l:30": [ { "axis0": [ 0, 1 ], "axis1": [ 5, 4 ], "matrix": "L22.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 5, 4 ], "matrix": "L22.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 124, 1 ], "matrix": "L22.mlp.W_out" }, { "axis0": [ 223, 1 ], "axis1": [ 5, 4 ], "matrix": "L22.mlp.W_in" }, { "axis0": [ 5, 4 ], "axis1": [ 124, 1 ], "matrix": "L22.mlp.W_out" } ], "l:300": [ { "axis0": [ 0, 1 ], "axis1": [ 1351, 200 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1351, 200 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 708, 17 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 120, 1 ], "axis1": [ 1351, 200 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 327, 1 ], "axis1": [ 1351, 200 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 525, 1 ], "axis1": [ 1351, 200 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 525, 2 ], "axis1": [ 1351, 200 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 526, 2 ], "axis1": [ 1351, 200 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 527, 2 ], "axis1": [ 1351, 200 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 528, 2 ], "axis1": [ 1351, 200 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 529, 2 ], "axis1": [ 1351, 200 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 530, 2 ], "axis1": [ 1351, 200 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 531, 2 ], "axis1": [ 1351, 200 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 532, 2 ], "axis1": [ 1351, 200 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 533, 2 ], "axis1": [ 1351, 200 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 534, 2 ], "axis1": [ 1351, 200 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 535, 2 ], "axis1": [ 1351, 200 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 536, 2 ], "axis1": [ 1351, 200 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 537, 2 ], "axis1": [ 1351, 200 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 538, 2 ], "axis1": [ 1351, 200 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 539, 2 ], "axis1": [ 1351, 200 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 540, 2 ], "axis1": [ 1351, 200 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 541, 1 ], "axis1": [ 1351, 200 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1351, 200 ], "axis1": [ 708, 17 ], "matrix": "L14.mlp.W_out" } ], "l:302": [ { "axis0": [ 0, 1 ], "axis1": [ 525, 17 ], "matrix": "L13.mlp.W_out" } ], "l:303": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L14.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L14.attn.W_K" }, { "axis0": [ 469, 3 ], "axis1": [ 0, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 120, 1 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 327, 1 ], "matrix": "L14.attn.W_O" } ], "l:304": [ { "axis0": [ 0, 1 ], "axis1": [ 1291, 9 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1291, 9 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 469, 3 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 264, 2 ], "axis1": [ 1291, 9 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 351, 1 ], "axis1": [ 1291, 9 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 598, 3 ], "axis1": [ 1291, 9 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 1291, 9 ], "axis1": [ 469, 3 ], "matrix": "L13.mlp.W_out" } ], "l:306": [ { "axis0": [ 0, 1 ], "axis1": [ 1401, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1401, 100 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 598, 3 ], "matrix": "L11.mlp.W_out" }, { "axis0": [ 18, 8 ], "axis1": [ 1401, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 55, 2 ], "axis1": [ 1401, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 59, 8 ], "axis1": [ 1401, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 68, 1 ], "axis1": [ 1401, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 102, 9 ], "axis1": [ 1401, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 137, 2 ], "axis1": [ 1401, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 173, 1 ], "axis1": [ 1401, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 326, 1 ], "axis1": [ 1401, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 328, 2 ], "axis1": [ 1401, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 1401, 100 ], "axis1": [ 598, 3 ], "matrix": "L11.mlp.W_out" } ], "l:308": [ { "axis0": [ 0, 1 ], "axis1": [ 13, 9 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 13, 9 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 264, 2 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 351, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 592, 6 ], "axis1": [ 13, 9 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 13, 9 ], "axis1": [ 264, 2 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 13, 9 ], "axis1": [ 351, 1 ], "matrix": "L12.mlp.W_out" } ], "l:31": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 1 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 1 ], "matrix": "L21.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L21.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 223, 1 ], "matrix": "L21.mlp.W_out" }, { "axis0": [ 40, 1 ], "axis1": [ 1, 1 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 44, 1 ], "axis1": [ 1, 1 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 198, 9 ], "axis1": [ 1, 1 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 216, 5 ], "axis1": [ 1, 1 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 222, 1 ], "axis1": [ 1, 1 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 1, 1 ], "axis1": [ 223, 1 ], "matrix": "L21.mlp.W_out" } ], "l:310": [ { "axis0": [ 0, 1 ], "axis1": [ 1201, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1201, 100 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 592, 3 ], "matrix": "L11.mlp.W_out" }, { "axis0": [ 42, 4 ], "axis1": [ 1201, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 51, 4 ], "axis1": [ 1201, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 67, 1 ], "axis1": [ 1201, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 213, 9 ], "axis1": [ 1201, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 223, 1 ], "axis1": [ 1201, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 236, 3 ], "axis1": [ 1201, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 258, 5 ], "axis1": [ 1201, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 266, 7 ], "axis1": [ 1201, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 1201, 100 ], "axis1": [ 592, 3 ], "matrix": "L11.mlp.W_out" } ], "l:312": [ { "axis0": [ 0, 1 ], "axis1": [ 1301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1301, 100 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 595, 3 ], "matrix": "L11.mlp.W_out" }, { "axis0": [ 49, 2 ], "axis1": [ 1301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 174, 16 ], "axis1": [ 1301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 210, 1 ], "axis1": [ 1301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 239, 8 ], "axis1": [ 1301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 248, 3 ], "axis1": [ 1301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 252, 4 ], "axis1": [ 1301, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 1301, 100 ], "axis1": [ 595, 3 ], "matrix": "L11.mlp.W_out" } ], "l:314": [ { "axis0": [ 0, 1 ], "axis1": [ 103, 17 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 103, 17 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 35, 1 ], "axis1": [ 103, 17 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 306, 17 ], "axis1": [ 103, 17 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 35, 1 ], "axis1": [ 103, 17 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 306, 17 ], "axis1": [ 103, 17 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 103, 17 ], "axis1": [ 46, 1 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 103, 17 ], "axis1": [ 198, 9 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 103, 17 ], "axis1": [ 224, 7 ], "matrix": "L5.mlp.W_out" } ], "l:316": [ { "axis0": [ 0, 1 ], "axis1": [ 218, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 218, 17 ], "matrix": "L19.mlp.W_up" }, { "axis0": [ 40, 1 ], "axis1": [ 218, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 60, 3 ], "axis1": [ 218, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 303, 1 ], "axis1": [ 218, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 305, 2 ], "axis1": [ 218, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 308, 4 ], "axis1": [ 218, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 313, 1 ], "axis1": [ 218, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 315, 1 ], "axis1": [ 218, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 338, 1 ], "axis1": [ 218, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 340, 1 ], "axis1": [ 218, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 362, 2 ], "axis1": [ 218, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 393, 1 ], "axis1": [ 218, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 40, 1 ], "axis1": [ 218, 17 ], "matrix": "L19.mlp.W_up" }, { "axis0": [ 60, 3 ], "axis1": [ 218, 17 ], "matrix": "L19.mlp.W_up" }, { "axis0": [ 303, 1 ], "axis1": [ 218, 17 ], "matrix": "L19.mlp.W_up" }, { "axis0": [ 305, 2 ], "axis1": [ 218, 17 ], "matrix": "L19.mlp.W_up" }, { "axis0": [ 308, 4 ], "axis1": [ 218, 17 ], "matrix": "L19.mlp.W_up" }, { "axis0": [ 313, 1 ], "axis1": [ 218, 17 ], "matrix": "L19.mlp.W_up" }, { "axis0": [ 315, 1 ], "axis1": [ 218, 17 ], "matrix": "L19.mlp.W_up" }, { "axis0": [ 338, 1 ], "axis1": [ 218, 17 ], "matrix": "L19.mlp.W_up" }, { "axis0": [ 340, 1 ], "axis1": [ 218, 17 ], "matrix": "L19.mlp.W_up" }, { "axis0": [ 362, 2 ], "axis1": [ 218, 17 ], "matrix": "L19.mlp.W_up" }, { "axis0": [ 393, 1 ], "axis1": [ 218, 17 ], "matrix": "L19.mlp.W_up" }, { "axis0": [ 218, 17 ], "axis1": [ 56, 1 ], "matrix": "L19.mlp.W_out" }, { "axis0": [ 218, 17 ], "axis1": [ 58, 2 ], "matrix": "L19.mlp.W_out" }, { "axis0": [ 218, 17 ], "axis1": [ 63, 1 ], "matrix": "L19.mlp.W_out" }, { "axis0": [ 218, 17 ], "axis1": [ 290, 9 ], "matrix": "L19.mlp.W_out" }, { "axis0": [ 218, 17 ], "axis1": [ 316, 1 ], "matrix": "L19.mlp.W_out" }, { "axis0": [ 218, 17 ], "axis1": [ 318, 1 ], "matrix": "L19.mlp.W_out" }, { "axis0": [ 218, 17 ], "axis1": [ 320, 1 ], "matrix": "L19.mlp.W_out" }, { "axis0": [ 218, 17 ], "axis1": [ 328, 1 ], "matrix": "L19.mlp.W_out" } ], "l:318": [ { "axis0": [ 0, 1 ], "axis1": [ 10, 200 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 10, 200 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 60, 3 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 303, 1 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 305, 2 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 308, 4 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 313, 1 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 315, 1 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 338, 1 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 340, 1 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 362, 2 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 393, 1 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 65, 1 ], "axis1": [ 10, 200 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 326, 1 ], "axis1": [ 10, 200 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 417, 34 ], "axis1": [ 10, 200 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 10, 200 ], "axis1": [ 60, 3 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 10, 200 ], "axis1": [ 303, 1 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 10, 200 ], "axis1": [ 305, 2 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 10, 200 ], "axis1": [ 308, 4 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 10, 200 ], "axis1": [ 313, 1 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 10, 200 ], "axis1": [ 315, 1 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 10, 200 ], "axis1": [ 338, 1 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 10, 200 ], "axis1": [ 340, 1 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 10, 200 ], "axis1": [ 362, 2 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 10, 200 ], "axis1": [ 393, 1 ], "matrix": "L18.mlp.W_out" } ], "l:32": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L21.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L21.attn.W_K" }, { "axis0": [ 28, 1 ], "axis1": [ 0, 64 ], "matrix": "L21.attn.W_V" }, { "axis0": [ 39, 1 ], "axis1": [ 0, 64 ], "matrix": "L21.attn.W_V" }, { "axis0": [ 49, 1 ], "axis1": [ 0, 64 ], "matrix": "L21.attn.W_V" }, { "axis0": [ 121, 16 ], "axis1": [ 0, 64 ], "matrix": "L21.attn.W_V" }, { "axis0": [ 139, 17 ], "axis1": [ 0, 64 ], "matrix": "L21.attn.W_V" }, { "axis0": [ 194, 3 ], "axis1": [ 0, 64 ], "matrix": "L21.attn.W_V" }, { "axis0": [ 240, 7 ], "axis1": [ 0, 64 ], "matrix": "L21.attn.W_V" }, { "axis0": [ 248, 3 ], "axis1": [ 0, 64 ], "matrix": "L21.attn.W_V" }, { "axis0": [ 252, 1 ], "axis1": [ 0, 64 ], "matrix": "L21.attn.W_V" }, { "axis0": [ 335, 1 ], "axis1": [ 0, 64 ], "matrix": "L21.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 40, 1 ], "matrix": "L21.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 44, 1 ], "matrix": "L21.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 198, 9 ], "matrix": "L21.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 216, 5 ], "matrix": "L21.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 222, 1 ], "matrix": "L21.attn.W_O" } ], "l:320": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L18.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L18.attn.W_K" }, { "axis0": [ 208, 1 ], "axis1": [ 0, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 334, 1 ], "axis1": [ 0, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 404, 1 ], "axis1": [ 0, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 65, 1 ], "matrix": "L18.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 326, 1 ], "matrix": "L18.attn.W_O" } ], "l:321": [ { "axis0": [ 0, 1 ], "axis1": [ 10, 9 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 10, 9 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 208, 1 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 334, 1 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 404, 1 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 44, 1 ], "axis1": [ 10, 9 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 220, 1 ], "axis1": [ 10, 9 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 222, 1 ], "axis1": [ 10, 9 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 340, 1 ], "axis1": [ 10, 9 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 362, 2 ], "axis1": [ 10, 9 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 10, 9 ], "axis1": [ 208, 1 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 10, 9 ], "axis1": [ 334, 1 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 10, 9 ], "axis1": [ 404, 1 ], "matrix": "L17.mlp.W_out" } ], "l:323": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 2 ], "matrix": "L23.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 2 ], "matrix": "L23.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L23.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L23.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 121, 1 ], "matrix": "L23.mlp.W_out" }, { "axis0": [ 125, 1 ], "axis1": [ 1, 2 ], "matrix": "L23.mlp.W_in" }, { "axis0": [ 1, 2 ], "axis1": [ 121, 1 ], "matrix": "L23.mlp.W_out" } ], "l:324": [ { "axis0": [ 0, 1 ], "axis1": [ 9, 2 ], "matrix": "L22.mlp.W_up" }, { "axis0": [ 121, 1 ], "axis1": [ 9, 1 ], "matrix": "L22.mlp.W_in" }, { "axis0": [ 225, 2 ], "axis1": [ 9, 1 ], "matrix": "L22.mlp.W_in" }, { "axis0": [ 254, 1 ], "axis1": [ 9, 1 ], "matrix": "L22.mlp.W_in" }, { "axis0": [ 121, 1 ], "axis1": [ 10, 1 ], "matrix": "L22.mlp.W_in" }, { "axis0": [ 225, 2 ], "axis1": [ 10, 1 ], "matrix": "L22.mlp.W_in" }, { "axis0": [ 254, 1 ], "axis1": [ 10, 1 ], "matrix": "L22.mlp.W_in" }, { "axis0": [ 9, 1 ], "axis1": [ 125, 1 ], "diagonal": true, "matrix": "L22.mlp.W_out" }, { "axis0": [ 10, 1 ], "axis1": [ 125, 1 ], "diagonal": true, "matrix": "L22.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 9, 1 ], "matrix": "L22.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 10, 1 ], "matrix": "L22.mlp.W_in" } ], "l:326": [ { "axis0": [ 0, 1 ], "axis1": [ 20, 2 ], "matrix": "L20.mlp.W_up" }, { "axis0": [ 321, 1 ], "axis1": [ 20, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 321, 1 ], "axis1": [ 21, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 20, 1 ], "axis1": [ 254, 1 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 21, 1 ], "axis1": [ 254, 1 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 254, 1 ], "matrix": "L20.mlp.W_out" } ], "l:327": [ { "axis0": [ 0, 1 ], "axis1": [ 235, 1 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 235, 1 ], "matrix": "L19.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 321, 1 ], "matrix": "L19.mlp.W_out" }, { "axis0": [ 35, 1 ], "axis1": [ 235, 1 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 42, 2 ], "axis1": [ 235, 1 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 188, 1 ], "axis1": [ 235, 1 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 207, 9 ], "axis1": [ 235, 1 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 327, 1 ], "axis1": [ 235, 1 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 333, 2 ], "axis1": [ 235, 1 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 404, 1 ], "axis1": [ 235, 1 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 235, 1 ], "axis1": [ 321, 1 ], "matrix": "L19.mlp.W_out" } ], "l:328": [ { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L19.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L19.attn.W_K" }, { "axis0": [ 18, 2 ], "axis1": [ 64, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 51, 1 ], "axis1": [ 64, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 55, 1 ], "axis1": [ 64, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 67, 1 ], "axis1": [ 64, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 258, 6 ], "axis1": [ 64, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 266, 6 ], "axis1": [ 64, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 329, 1 ], "axis1": [ 64, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 336, 1 ], "axis1": [ 64, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 342, 1 ], "axis1": [ 64, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 344, 2 ], "axis1": [ 64, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 391, 1 ], "axis1": [ 64, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 405, 9 ], "axis1": [ 64, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 415, 2 ], "axis1": [ 64, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 468, 4 ], "axis1": [ 64, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 490, 12 ], "axis1": [ 64, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 510, 1 ], "axis1": [ 64, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 35, 1 ], "matrix": "L19.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 42, 2 ], "matrix": "L19.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 188, 1 ], "matrix": "L19.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 207, 9 ], "matrix": "L19.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 327, 1 ], "matrix": "L19.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 333, 2 ], "matrix": "L19.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 404, 1 ], "matrix": "L19.attn.W_O" } ], "l:330": [ { "axis0": [ 0, 1 ], "axis1": [ 618, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 618, 17 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 33, 1 ], "axis1": [ 618, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 691, 17 ], "axis1": [ 618, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 33, 1 ], "axis1": [ 618, 17 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 691, 17 ], "axis1": [ 618, 17 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 618, 17 ], "axis1": [ 51, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 618, 17 ], "axis1": [ 55, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 618, 17 ], "axis1": [ 67, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 618, 17 ], "axis1": [ 258, 6 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 618, 17 ], "axis1": [ 266, 6 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 618, 17 ], "axis1": [ 336, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 618, 17 ], "axis1": [ 391, 1 ], "matrix": "L15.mlp.W_out" } ], "l:332": [ { "axis0": [ 0, 1 ], "axis1": [ 1151, 200 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1151, 200 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 691, 17 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 49, 2 ], "axis1": [ 1151, 200 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 174, 16 ], "axis1": [ 1151, 200 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 210, 1 ], "axis1": [ 1151, 200 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 239, 8 ], "axis1": [ 1151, 200 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 248, 3 ], "axis1": [ 1151, 200 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 252, 4 ], "axis1": [ 1151, 200 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 523, 2 ], "axis1": [ 1151, 200 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1151, 200 ], "axis1": [ 691, 17 ], "matrix": "L14.mlp.W_out" } ], "l:334": [ { "axis0": [ 0, 1 ], "axis1": [ 1846, 4 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 352, 3 ], "axis1": [ 1846, 2 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 352, 3 ], "axis1": [ 1848, 2 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 1846, 2 ], "axis1": [ 523, 2 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 1848, 2 ], "axis1": [ 523, 2 ], "diagonal": true, "matrix": "L13.mlp.W_out" } ], "l:335": [ { "axis0": [ 0, 1 ], "axis1": [ 22, 9 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 22, 9 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 352, 3 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 592, 3 ], "axis1": [ 22, 9 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 598, 3 ], "axis1": [ 22, 9 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 22, 9 ], "axis1": [ 352, 3 ], "matrix": "L12.mlp.W_out" } ], "l:337": [ { "axis0": [ 0, 1 ], "axis1": [ 410, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 410, 17 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 35, 1 ], "axis1": [ 410, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 42, 1 ], "axis1": [ 410, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 45, 1 ], "axis1": [ 410, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 210, 1 ], "axis1": [ 410, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 212, 8 ], "axis1": [ 410, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 223, 1 ], "axis1": [ 410, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 236, 4 ], "axis1": [ 410, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 256, 1 ], "axis1": [ 410, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 35, 1 ], "axis1": [ 410, 17 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 42, 1 ], "axis1": [ 410, 17 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 45, 1 ], "axis1": [ 410, 17 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 210, 1 ], "axis1": [ 410, 17 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 212, 8 ], "axis1": [ 410, 17 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 223, 1 ], "axis1": [ 410, 17 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 236, 4 ], "axis1": [ 410, 17 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 256, 1 ], "axis1": [ 410, 17 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 410, 17 ], "axis1": [ 18, 1 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 410, 17 ], "axis1": [ 329, 1 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 410, 17 ], "axis1": [ 342, 1 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 410, 17 ], "axis1": [ 344, 1 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 410, 17 ], "axis1": [ 405, 9 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 410, 17 ], "axis1": [ 415, 2 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 410, 17 ], "axis1": [ 468, 2 ], "matrix": "L18.mlp.W_out" } ], "l:339": [ { "axis0": [ 0, 1 ], "axis1": [ 28, 200 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 28, 200 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 42, 1 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 45, 1 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 210, 1 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 212, 8 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 223, 1 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 236, 4 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 256, 1 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 65, 1 ], "axis1": [ 28, 200 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 326, 1 ], "axis1": [ 28, 200 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 657, 34 ], "axis1": [ 28, 200 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 28, 200 ], "axis1": [ 42, 1 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 28, 200 ], "axis1": [ 45, 1 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 28, 200 ], "axis1": [ 210, 1 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 28, 200 ], "axis1": [ 212, 8 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 28, 200 ], "axis1": [ 223, 1 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 28, 200 ], "axis1": [ 236, 4 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 28, 200 ], "axis1": [ 256, 1 ], "matrix": "L17.mlp.W_out" } ], "l:34": [ { "axis0": [ 0, 1 ], "axis1": [ 60, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 60, 17 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 33, 1 ], "axis1": [ 60, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 69, 17 ], "axis1": [ 60, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 33, 1 ], "axis1": [ 60, 17 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 69, 17 ], "axis1": [ 60, 17 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 60, 17 ], "axis1": [ 139, 17 ], "matrix": "L4.mlp.W_out" } ], "l:341": [ { "axis0": [ 0, 1 ], "axis1": [ 1083, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1083, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 49, 2 ], "axis1": [ 1083, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 174, 16 ], "axis1": [ 1083, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 210, 1 ], "axis1": [ 1083, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 239, 8 ], "axis1": [ 1083, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 248, 3 ], "axis1": [ 1083, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 252, 4 ], "axis1": [ 1083, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 468, 1 ], "axis1": [ 1083, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 49, 2 ], "axis1": [ 1083, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 174, 16 ], "axis1": [ 1083, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 210, 1 ], "axis1": [ 1083, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 239, 8 ], "axis1": [ 1083, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 248, 3 ], "axis1": [ 1083, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 252, 4 ], "axis1": [ 1083, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 468, 1 ], "axis1": [ 1083, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1083, 34 ], "axis1": [ 657, 17 ], "matrix": "L14.mlp.W_out" } ], "l:343": [ { "axis0": [ 0, 1 ], "axis1": [ 1289, 2 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1289, 2 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 468, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 58, 1 ], "axis1": [ 1289, 2 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 1289, 2 ], "axis1": [ 468, 1 ], "matrix": "L13.mlp.W_out" } ], "l:344": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L13.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L13.attn.W_K" }, { "axis0": [ 209, 1 ], "axis1": [ 0, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 211, 2 ], "axis1": [ 0, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 222, 1 ], "axis1": [ 0, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 256, 1 ], "axis1": [ 0, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 263, 1 ], "axis1": [ 0, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 58, 1 ], "matrix": "L13.attn.W_O" } ], "l:346": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 589, 1 ], "axis1": [ 1, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 1, 2 ], "axis1": [ 209, 1 ], "matrix": "L12.mlp.W_out" } ], "l:347": [ { "axis0": [ 0, 1 ], "axis1": [ 901, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 901, 100 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 49, 2 ], "axis1": [ 901, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 174, 16 ], "axis1": [ 901, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 210, 1 ], "axis1": [ 901, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 239, 8 ], "axis1": [ 901, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 248, 3 ], "axis1": [ 901, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 252, 4 ], "axis1": [ 901, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 901, 100 ], "axis1": [ 589, 1 ], "matrix": "L11.mlp.W_out" } ], "l:349": [ { "axis0": [ 0, 1 ], "axis1": [ 3, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 3, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 211, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 589, 1 ], "axis1": [ 3, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 3, 2 ], "axis1": [ 211, 1 ], "matrix": "L12.mlp.W_out" } ], "l:350": [ { "axis0": [ 0, 1 ], "axis1": [ 5, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 5, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 590, 1 ], "axis1": [ 5, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 5, 2 ], "axis1": [ 212, 1 ], "matrix": "L12.mlp.W_out" } ], "l:351": [ { "axis0": [ 0, 1 ], "axis1": [ 1001, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1001, 100 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 42, 4 ], "axis1": [ 1001, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 51, 4 ], "axis1": [ 1001, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 67, 1 ], "axis1": [ 1001, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 213, 9 ], "axis1": [ 1001, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 223, 1 ], "axis1": [ 1001, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 236, 3 ], "axis1": [ 1001, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 258, 5 ], "axis1": [ 1001, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 266, 7 ], "axis1": [ 1001, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 1001, 100 ], "axis1": [ 590, 1 ], "matrix": "L11.mlp.W_out" } ], "l:353": [ { "axis0": [ 0, 1 ], "axis1": [ 7, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 7, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 222, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 590, 1 ], "axis1": [ 7, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 7, 2 ], "axis1": [ 222, 1 ], "matrix": "L12.mlp.W_out" } ], "l:354": [ { "axis0": [ 0, 1 ], "axis1": [ 9, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 9, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 591, 1 ], "axis1": [ 9, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 9, 2 ], "axis1": [ 256, 1 ], "matrix": "L12.mlp.W_out" } ], "l:355": [ { "axis0": [ 0, 1 ], "axis1": [ 1101, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1101, 100 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 18, 8 ], "axis1": [ 1101, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 55, 2 ], "axis1": [ 1101, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 59, 8 ], "axis1": [ 1101, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 68, 1 ], "axis1": [ 1101, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 102, 9 ], "axis1": [ 1101, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 137, 2 ], "axis1": [ 1101, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 173, 1 ], "axis1": [ 1101, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 326, 1 ], "axis1": [ 1101, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 328, 2 ], "axis1": [ 1101, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 1101, 100 ], "axis1": [ 591, 1 ], "matrix": "L11.mlp.W_out" } ], "l:357": [ { "axis0": [ 0, 1 ], "axis1": [ 11, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 11, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 263, 1 ], "matrix": "L12.mlp.W_out" }, { "axis0": [ 591, 1 ], "axis1": [ 11, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 11, 2 ], "axis1": [ 263, 1 ], "matrix": "L12.mlp.W_out" } ], "l:358": [ { "axis0": [ 0, 1 ], "axis1": [ 1117, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1117, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 49, 2 ], "axis1": [ 1117, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 174, 16 ], "axis1": [ 1117, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 210, 1 ], "axis1": [ 1117, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 239, 8 ], "axis1": [ 1117, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 248, 3 ], "axis1": [ 1117, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 252, 4 ], "axis1": [ 1117, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 468, 1 ], "axis1": [ 1117, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 49, 2 ], "axis1": [ 1117, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 174, 16 ], "axis1": [ 1117, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 210, 1 ], "axis1": [ 1117, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 239, 8 ], "axis1": [ 1117, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 248, 3 ], "axis1": [ 1117, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 252, 4 ], "axis1": [ 1117, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 468, 1 ], "axis1": [ 1117, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1117, 34 ], "axis1": [ 674, 17 ], "matrix": "L14.mlp.W_out" } ], "l:36": [ { "axis0": [ 60, 2 ], "axis1": [ 192, 64 ], "matrix": "L4.attn.W_Q" }, { "axis0": [ 27, 1 ], "axis1": [ 192, 64 ], "matrix": "L4.attn.W_K" }, { "axis0": [ 62, 1 ], "axis1": [ 192, 64 ], "matrix": "L4.attn.W_K" }, { "axis0": [ 18, 1 ], "axis1": [ 192, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 192, 64 ], "axis1": [ 33, 1 ], "matrix": "L4.attn.W_O" } ], "l:360": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L17.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L17.attn.W_K" }, { "axis0": [ 374, 3 ], "axis1": [ 0, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 65, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 326, 1 ], "matrix": "L17.attn.W_O" } ], "l:361": [ { "axis0": [ 0, 1 ], "axis1": [ 37, 9 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 37, 9 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 374, 3 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 45, 1 ], "axis1": [ 37, 9 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 223, 1 ], "axis1": [ 37, 9 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 236, 4 ], "axis1": [ 37, 9 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 37, 9 ], "axis1": [ 374, 3 ], "matrix": "L16.mlp.W_out" } ], "l:363": [ { "axis0": [ 0, 1 ], "axis1": [ 401, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 401, 100 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 45, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 223, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 239, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 606, 17 ], "axis1": [ 401, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 640, 17 ], "axis1": [ 401, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 401, 100 ], "axis1": [ 45, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 401, 100 ], "axis1": [ 223, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 401, 100 ], "axis1": [ 239, 1 ], "matrix": "L15.mlp.W_out" } ], "l:365": [ { "axis0": [ 0, 1 ], "axis1": [ 981, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 981, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 18, 8 ], "axis1": [ 981, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 55, 2 ], "axis1": [ 981, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 59, 8 ], "axis1": [ 981, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 68, 1 ], "axis1": [ 981, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 102, 9 ], "axis1": [ 981, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 137, 2 ], "axis1": [ 981, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 173, 1 ], "axis1": [ 981, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 326, 1 ], "axis1": [ 981, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 328, 2 ], "axis1": [ 981, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 468, 1 ], "axis1": [ 981, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 18, 8 ], "axis1": [ 981, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 55, 2 ], "axis1": [ 981, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 59, 8 ], "axis1": [ 981, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 68, 1 ], "axis1": [ 981, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 102, 9 ], "axis1": [ 981, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 137, 2 ], "axis1": [ 981, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 173, 1 ], "axis1": [ 981, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 326, 1 ], "axis1": [ 981, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 328, 2 ], "axis1": [ 981, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 468, 1 ], "axis1": [ 981, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 981, 34 ], "axis1": [ 606, 17 ], "matrix": "L14.mlp.W_out" } ], "l:367": [ { "axis0": [ 0, 1 ], "axis1": [ 1049, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1049, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 18, 8 ], "axis1": [ 1049, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 55, 2 ], "axis1": [ 1049, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 59, 8 ], "axis1": [ 1049, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 68, 1 ], "axis1": [ 1049, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 102, 9 ], "axis1": [ 1049, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 137, 2 ], "axis1": [ 1049, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 173, 1 ], "axis1": [ 1049, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 326, 1 ], "axis1": [ 1049, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 328, 2 ], "axis1": [ 1049, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 468, 1 ], "axis1": [ 1049, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 18, 8 ], "axis1": [ 1049, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 55, 2 ], "axis1": [ 1049, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 59, 8 ], "axis1": [ 1049, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 68, 1 ], "axis1": [ 1049, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 102, 9 ], "axis1": [ 1049, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 137, 2 ], "axis1": [ 1049, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 173, 1 ], "axis1": [ 1049, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 326, 1 ], "axis1": [ 1049, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 328, 2 ], "axis1": [ 1049, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 468, 1 ], "axis1": [ 1049, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1049, 34 ], "axis1": [ 640, 17 ], "matrix": "L14.mlp.W_out" } ], "l:369": [ { "axis0": [ 0, 1 ], "axis1": [ 501, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 501, 100 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 236, 3 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 116, 4 ], "axis1": [ 501, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 349, 1 ], "axis1": [ 501, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 580, 4 ], "axis1": [ 501, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 585, 3 ], "axis1": [ 501, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 601, 5 ], "axis1": [ 501, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 623, 17 ], "axis1": [ 501, 100 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 501, 100 ], "axis1": [ 236, 3 ], "matrix": "L15.mlp.W_out" } ], "l:371": [ { "axis0": [ 0, 1 ], "axis1": [ 947, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 947, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 42, 4 ], "axis1": [ 947, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 51, 4 ], "axis1": [ 947, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 67, 1 ], "axis1": [ 947, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 213, 9 ], "axis1": [ 947, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 223, 1 ], "axis1": [ 947, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 236, 3 ], "axis1": [ 947, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 258, 5 ], "axis1": [ 947, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 266, 7 ], "axis1": [ 947, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 468, 1 ], "axis1": [ 947, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 42, 4 ], "axis1": [ 947, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 51, 4 ], "axis1": [ 947, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 67, 1 ], "axis1": [ 947, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 213, 9 ], "axis1": [ 947, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 223, 1 ], "axis1": [ 947, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 236, 3 ], "axis1": [ 947, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 258, 5 ], "axis1": [ 947, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 266, 7 ], "axis1": [ 947, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 468, 1 ], "axis1": [ 947, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 947, 34 ], "axis1": [ 116, 4 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 947, 34 ], "axis1": [ 349, 1 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 947, 34 ], "axis1": [ 580, 4 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 947, 34 ], "axis1": [ 585, 3 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 947, 34 ], "axis1": [ 601, 5 ], "matrix": "L14.mlp.W_out" } ], "l:373": [ { "axis0": [ 0, 1 ], "axis1": [ 1015, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1015, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 42, 4 ], "axis1": [ 1015, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 51, 4 ], "axis1": [ 1015, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 67, 1 ], "axis1": [ 1015, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 213, 9 ], "axis1": [ 1015, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 223, 1 ], "axis1": [ 1015, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 236, 3 ], "axis1": [ 1015, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 258, 5 ], "axis1": [ 1015, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 266, 7 ], "axis1": [ 1015, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 468, 1 ], "axis1": [ 1015, 34 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 42, 4 ], "axis1": [ 1015, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 51, 4 ], "axis1": [ 1015, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 67, 1 ], "axis1": [ 1015, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 213, 9 ], "axis1": [ 1015, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 223, 1 ], "axis1": [ 1015, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 236, 3 ], "axis1": [ 1015, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 258, 5 ], "axis1": [ 1015, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 266, 7 ], "axis1": [ 1015, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 468, 1 ], "axis1": [ 1015, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1015, 34 ], "axis1": [ 623, 17 ], "matrix": "L14.mlp.W_out" } ], "l:375": [ { "axis0": [ 0, 1 ], "axis1": [ 427, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 427, 17 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 40, 1 ], "axis1": [ 427, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 52, 3 ], "axis1": [ 427, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 56, 1 ], "axis1": [ 427, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 58, 2 ], "axis1": [ 427, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 264, 2 ], "axis1": [ 427, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 290, 9 ], "axis1": [ 427, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 40, 1 ], "axis1": [ 427, 17 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 52, 3 ], "axis1": [ 427, 17 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 56, 1 ], "axis1": [ 427, 17 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 58, 2 ], "axis1": [ 427, 17 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 264, 2 ], "axis1": [ 427, 17 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 290, 9 ], "axis1": [ 427, 17 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 427, 17 ], "axis1": [ 19, 1 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 427, 17 ], "axis1": [ 345, 1 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 427, 17 ], "axis1": [ 470, 2 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 427, 17 ], "axis1": [ 490, 12 ], "matrix": "L18.mlp.W_out" }, { "axis0": [ 427, 17 ], "axis1": [ 510, 1 ], "matrix": "L18.mlp.W_out" } ], "l:377": [ { "axis0": [ 0, 1 ], "axis1": [ 228, 200 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 228, 200 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 52, 3 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 56, 1 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 58, 2 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 264, 2 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 290, 9 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 43, 1 ], "axis1": [ 228, 200 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 60, 1 ], "axis1": [ 228, 200 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 111, 1 ], "axis1": [ 228, 200 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 342, 1 ], "axis1": [ 228, 200 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 399, 5 ], "axis1": [ 228, 200 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 405, 9 ], "axis1": [ 228, 200 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 415, 2 ], "axis1": [ 228, 200 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 543, 16 ], "axis1": [ 228, 200 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 228, 200 ], "axis1": [ 52, 3 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 228, 200 ], "axis1": [ 56, 1 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 228, 200 ], "axis1": [ 58, 2 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 228, 200 ], "axis1": [ 264, 2 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 228, 200 ], "axis1": [ 290, 9 ], "matrix": "L17.mlp.W_out" } ], "l:379": [ { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L17.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L17.attn.W_K" }, { "axis0": [ 368, 3 ], "axis1": [ 64, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 43, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 60, 1 ], "matrix": "L17.attn.W_O" } ], "l:38": [ { "axis0": [ 0, 1 ], "axis1": [ 60, 1 ], "matrix": "L3.mlp.W_out" } ], "l:380": [ { "axis0": [ 0, 1 ], "axis1": [ 3, 2 ], "matrix": "L23.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 3, 2 ], "matrix": "L23.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 127, 1 ], "matrix": "L23.mlp.W_out" }, { "axis0": [ 126, 1 ], "axis1": [ 3, 2 ], "matrix": "L23.mlp.W_in" }, { "axis0": [ 3, 2 ], "axis1": [ 127, 1 ], "matrix": "L23.mlp.W_out" } ], "l:381": [ { "axis0": [ 0, 1 ], "axis1": [ 11, 2 ], "matrix": "L22.mlp.W_up" }, { "axis0": [ 121, 1 ], "axis1": [ 11, 1 ], "matrix": "L22.mlp.W_in" }, { "axis0": [ 224, 3 ], "axis1": [ 11, 1 ], "matrix": "L22.mlp.W_in" }, { "axis0": [ 254, 1 ], "axis1": [ 11, 1 ], "matrix": "L22.mlp.W_in" }, { "axis0": [ 121, 1 ], "axis1": [ 12, 1 ], "matrix": "L22.mlp.W_in" }, { "axis0": [ 224, 3 ], "axis1": [ 12, 1 ], "matrix": "L22.mlp.W_in" }, { "axis0": [ 254, 1 ], "axis1": [ 12, 1 ], "matrix": "L22.mlp.W_in" }, { "axis0": [ 11, 1 ], "axis1": [ 126, 1 ], "diagonal": true, "matrix": "L22.mlp.W_out" }, { "axis0": [ 12, 1 ], "axis1": [ 126, 1 ], "diagonal": true, "matrix": "L22.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 11, 1 ], "matrix": "L22.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 12, 1 ], "matrix": "L22.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 126, 1 ], "matrix": "L22.mlp.W_out" } ], "l:383": [ { "axis0": [ 0, 1 ], "axis1": [ 2, 1 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 2, 1 ], "matrix": "L21.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 224, 1 ], "matrix": "L21.mlp.W_out" }, { "axis0": [ 57, 1 ], "axis1": [ 2, 1 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 274, 11 ], "axis1": [ 2, 1 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 288, 2 ], "axis1": [ 2, 1 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 302, 1 ], "axis1": [ 2, 1 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 337, 1 ], "axis1": [ 2, 1 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 339, 1 ], "axis1": [ 2, 1 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 2, 1 ], "axis1": [ 224, 1 ], "matrix": "L21.mlp.W_out" } ], "l:384": [ { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L18.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L18.attn.W_K" }, { "axis0": [ 57, 1 ], "axis1": [ 128, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 61, 4 ], "axis1": [ 128, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 68, 1 ], "axis1": [ 128, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 273, 12 ], "axis1": [ 128, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 288, 2 ], "axis1": [ 128, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 299, 5 ], "axis1": [ 128, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 305, 2 ], "axis1": [ 128, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 308, 4 ], "axis1": [ 128, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 313, 1 ], "axis1": [ 128, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 315, 2 ], "axis1": [ 128, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 318, 1 ], "axis1": [ 128, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 320, 3 ], "axis1": [ 128, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 328, 1 ], "axis1": [ 128, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 337, 3 ], "axis1": [ 128, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 351, 1 ], "axis1": [ 128, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 354, 1 ], "axis1": [ 128, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 356, 6 ], "axis1": [ 128, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 393, 1 ], "axis1": [ 128, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 57, 1 ], "matrix": "L18.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 274, 11 ], "matrix": "L18.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 288, 2 ], "matrix": "L18.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 302, 1 ], "matrix": "L18.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 337, 1 ], "matrix": "L18.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 339, 1 ], "matrix": "L18.attn.W_O" } ], "l:386": [ { "axis0": [ 0, 1 ], "axis1": [ 635, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 635, 17 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 33, 1 ], "axis1": [ 635, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 345, 1 ], "axis1": [ 635, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 490, 16 ], "axis1": [ 635, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 33, 1 ], "axis1": [ 635, 17 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 345, 1 ], "axis1": [ 635, 17 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 490, 16 ], "axis1": [ 635, 17 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 635, 17 ], "axis1": [ 57, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 635, 17 ], "axis1": [ 273, 12 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 635, 17 ], "axis1": [ 288, 2 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 635, 17 ], "axis1": [ 337, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 635, 17 ], "axis1": [ 339, 1 ], "matrix": "L15.mlp.W_out" } ], "l:388": [ { "axis0": [ 0, 1 ], "axis1": [ 1446, 200 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1446, 200 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 345, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 490, 16 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 42, 4 ], "axis1": [ 1446, 200 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 51, 4 ], "axis1": [ 1446, 200 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 67, 1 ], "axis1": [ 1446, 200 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 213, 9 ], "axis1": [ 1446, 200 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 223, 1 ], "axis1": [ 1446, 200 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 236, 3 ], "axis1": [ 1446, 200 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 258, 5 ], "axis1": [ 1446, 200 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 266, 7 ], "axis1": [ 1446, 200 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 358, 2 ], "axis1": [ 1446, 200 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 1446, 200 ], "axis1": [ 345, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 1446, 200 ], "axis1": [ 490, 16 ], "matrix": "L13.mlp.W_out" } ], "l:39": [ { "axis0": [ 0, 1 ], "axis1": [ 61, 1 ], "matrix": "L3.mlp.W_out" } ], "l:390": [ { "axis0": [ 0, 1 ], "axis1": [ 37, 4 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 598, 3 ], "axis1": [ 37, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 598, 3 ], "axis1": [ 39, 2 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 37, 2 ], "axis1": [ 358, 2 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 39, 2 ], "axis1": [ 358, 2 ], "diagonal": true, "matrix": "L12.mlp.W_out" } ], "l:391": [ { "axis0": [ 0, 1 ], "axis1": [ 428, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 428, 17 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 18, 2 ], "axis1": [ 428, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 35, 1 ], "axis1": [ 428, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 329, 1 ], "axis1": [ 428, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 344, 2 ], "axis1": [ 428, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 468, 4 ], "axis1": [ 428, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 490, 8 ], "axis1": [ 428, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 18, 2 ], "axis1": [ 428, 17 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 35, 1 ], "axis1": [ 428, 17 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 329, 1 ], "axis1": [ 428, 17 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 344, 2 ], "axis1": [ 428, 17 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 468, 4 ], "axis1": [ 428, 17 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 490, 8 ], "axis1": [ 428, 17 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 428, 17 ], "axis1": [ 61, 2 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 428, 17 ], "axis1": [ 299, 5 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 428, 17 ], "axis1": [ 305, 2 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 428, 17 ], "axis1": [ 308, 4 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 428, 17 ], "axis1": [ 313, 1 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 428, 17 ], "axis1": [ 315, 1 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 428, 17 ], "axis1": [ 338, 1 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 428, 17 ], "axis1": [ 393, 1 ], "matrix": "L17.mlp.W_out" } ], "l:393": [ { "axis0": [ 0, 1 ], "axis1": [ 88, 200 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 88, 200 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 18, 2 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 329, 1 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 344, 2 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 468, 4 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 490, 8 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 33, 1 ], "axis1": [ 88, 200 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 116, 4 ], "axis1": [ 88, 200 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 327, 1 ], "axis1": [ 88, 200 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 349, 1 ], "axis1": [ 88, 200 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 580, 4 ], "axis1": [ 88, 200 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 585, 3 ], "axis1": [ 88, 200 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 601, 5 ], "axis1": [ 88, 200 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 623, 17 ], "axis1": [ 88, 200 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 88, 200 ], "axis1": [ 18, 2 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 88, 200 ], "axis1": [ 329, 1 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 88, 200 ], "axis1": [ 344, 2 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 88, 200 ], "axis1": [ 468, 4 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 88, 200 ], "axis1": [ 490, 8 ], "matrix": "L16.mlp.W_out" } ], "l:395": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L16.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L16.attn.W_K" }, { "axis0": [ 45, 1 ], "axis1": [ 0, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 223, 1 ], "axis1": [ 0, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 239, 1 ], "axis1": [ 0, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 33, 1 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 0, 64 ], "axis1": [ 327, 1 ], "matrix": "L16.attn.W_O" } ], "l:396": [ { "axis0": [ 0, 1 ], "axis1": [ 445, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 445, 17 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 20, 1 ], "axis1": [ 445, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 40, 1 ], "axis1": [ 445, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 102, 1 ], "axis1": [ 445, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 346, 1 ], "axis1": [ 445, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 498, 4 ], "axis1": [ 445, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 505, 6 ], "axis1": [ 445, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 512, 3 ], "axis1": [ 445, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 519, 1 ], "axis1": [ 445, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 20, 1 ], "axis1": [ 445, 17 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 40, 1 ], "axis1": [ 445, 17 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 102, 1 ], "axis1": [ 445, 17 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 346, 1 ], "axis1": [ 445, 17 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 498, 4 ], "axis1": [ 445, 17 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 505, 6 ], "axis1": [ 445, 17 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 512, 3 ], "axis1": [ 445, 17 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 519, 1 ], "axis1": [ 445, 17 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 445, 17 ], "axis1": [ 63, 2 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 445, 17 ], "axis1": [ 68, 1 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 445, 17 ], "axis1": [ 316, 1 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 445, 17 ], "axis1": [ 318, 1 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 445, 17 ], "axis1": [ 320, 3 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 445, 17 ], "axis1": [ 328, 1 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 445, 17 ], "axis1": [ 351, 1 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 445, 17 ], "axis1": [ 354, 1 ], "matrix": "L17.mlp.W_out" }, { "axis0": [ 445, 17 ], "axis1": [ 356, 6 ], "matrix": "L17.mlp.W_out" } ], "l:398": [ { "axis0": [ 0, 1 ], "axis1": [ 288, 200 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 288, 200 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 20, 1 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 102, 1 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 346, 1 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 498, 4 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 505, 6 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 512, 3 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 519, 1 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 52, 3 ], "axis1": [ 288, 200 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 56, 1 ], "axis1": [ 288, 200 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 58, 6 ], "axis1": [ 288, 200 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 65, 1 ], "axis1": [ 288, 200 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 68, 1 ], "axis1": [ 288, 200 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 173, 4 ], "axis1": [ 288, 200 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 179, 1 ], "axis1": [ 288, 200 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 207, 1 ], "axis1": [ 288, 200 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 209, 1 ], "axis1": [ 288, 200 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 212, 1 ], "axis1": [ 288, 200 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 256, 1 ], "axis1": [ 288, 200 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 264, 2 ], "axis1": [ 288, 200 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 326, 1 ], "axis1": [ 288, 200 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 351, 11 ], "axis1": [ 288, 200 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 542, 1 ], "axis1": [ 288, 200 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 288, 200 ], "axis1": [ 20, 1 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 288, 200 ], "axis1": [ 102, 1 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 288, 200 ], "axis1": [ 346, 1 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 288, 200 ], "axis1": [ 498, 4 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 288, 200 ], "axis1": [ 505, 6 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 288, 200 ], "axis1": [ 512, 3 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 288, 200 ], "axis1": [ 519, 1 ], "matrix": "L16.mlp.W_out" } ], "l:4": [ { "axis0": [ 0, 1 ], "axis1": [ 39, 1 ], "matrix": "L0.mlp.W_out" } ], "l:400": [ { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L16.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L16.attn.W_K" }, { "axis0": [ 44, 1 ], "axis1": [ 64, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 220, 1 ], "axis1": [ 64, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 222, 1 ], "axis1": [ 64, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 68, 1 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 356, 1 ], "matrix": "L16.attn.W_O" } ], "l:401": [ { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L16.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L16.attn.W_K" }, { "axis0": [ 68, 1 ], "axis1": [ 128, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 364, 1 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 364, 1 ], "matrix": "L16.mlp.W_out" } ], "l:402": [ { "axis0": [ 0, 1 ], "axis1": [ 1052, 2 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1052, 2 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 68, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 35, 1 ], "axis1": [ 1052, 2 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 725, 1 ], "axis1": [ 1052, 2 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1052, 2 ], "axis1": [ 68, 1 ], "matrix": "L15.mlp.W_out" } ], "l:404": [ { "axis0": [ 0, 1 ], "axis1": [ 1551, 2 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1551, 2 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 725, 1 ], "matrix": "L14.mlp.W_out" }, { "axis0": [ 468, 1 ], "axis1": [ 1551, 2 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1551, 2 ], "axis1": [ 725, 1 ], "matrix": "L14.mlp.W_out" } ], "l:406": [ { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L17.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L17.attn.W_K" }, { "axis0": [ 21, 3 ], "axis1": [ 128, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 103, 8 ], "axis1": [ 128, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 120, 1 ], "axis1": [ 128, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 347, 1 ], "axis1": [ 128, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 395, 1 ], "axis1": [ 128, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 515, 4 ], "axis1": [ 128, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 520, 2 ], "axis1": [ 128, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 525, 9 ], "axis1": [ 128, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 535, 3 ], "axis1": [ 128, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 540, 2 ], "axis1": [ 128, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 726, 17 ], "axis1": [ 128, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 33, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 173, 15 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 392, 1 ], "matrix": "L17.attn.W_O" } ], "l:408": [ { "axis0": [ 0, 1 ], "axis1": [ 1553, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1553, 17 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 33, 1 ], "axis1": [ 1553, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 346, 1 ], "axis1": [ 1553, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 506, 5 ], "axis1": [ 1553, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 512, 11 ], "axis1": [ 1553, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 33, 1 ], "axis1": [ 1553, 17 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 346, 1 ], "axis1": [ 1553, 17 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 506, 5 ], "axis1": [ 1553, 17 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 512, 11 ], "axis1": [ 1553, 17 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 1553, 17 ], "axis1": [ 726, 17 ], "matrix": "L14.mlp.W_out" } ], "l:41": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 27, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 25, 1 ], "axis1": [ 1, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 40, 1 ], "axis1": [ 1, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 1, 2 ], "axis1": [ 27, 1 ], "matrix": "L3.mlp.W_out" } ], "l:410": [ { "axis0": [ 0, 1 ], "axis1": [ 1646, 200 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1646, 200 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 346, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 506, 5 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 512, 11 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 18, 8 ], "axis1": [ 1646, 200 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 55, 2 ], "axis1": [ 1646, 200 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 59, 8 ], "axis1": [ 1646, 200 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 68, 1 ], "axis1": [ 1646, 200 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 102, 9 ], "axis1": [ 1646, 200 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 137, 2 ], "axis1": [ 1646, 200 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 173, 1 ], "axis1": [ 1646, 200 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 326, 1 ], "axis1": [ 1646, 200 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 328, 2 ], "axis1": [ 1646, 200 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 360, 2 ], "axis1": [ 1646, 200 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 1646, 200 ], "axis1": [ 346, 1 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 1646, 200 ], "axis1": [ 506, 5 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 1646, 200 ], "axis1": [ 512, 11 ], "matrix": "L13.mlp.W_out" } ], "l:412": [ { "axis0": [ 0, 1 ], "axis1": [ 360, 2 ], "matrix": "L12.mlp.W_out" } ], "l:413": [ { "axis0": [ 0, 1 ], "axis1": [ 488, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 488, 17 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 35, 1 ], "axis1": [ 488, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 290, 14 ], "axis1": [ 488, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 305, 2 ], "axis1": [ 488, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 393, 1 ], "axis1": [ 488, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 35, 1 ], "axis1": [ 488, 17 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 290, 14 ], "axis1": [ 488, 17 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 305, 2 ], "axis1": [ 488, 17 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 393, 1 ], "axis1": [ 488, 17 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 488, 17 ], "axis1": [ 21, 1 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 488, 17 ], "axis1": [ 103, 3 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 488, 17 ], "axis1": [ 515, 4 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 488, 17 ], "axis1": [ 520, 2 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 488, 17 ], "axis1": [ 525, 7 ], "matrix": "L16.mlp.W_out" } ], "l:415": [ { "axis0": [ 0, 1 ], "axis1": [ 652, 200 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 652, 200 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 290, 14 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 305, 2 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 393, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 606, 17 ], "axis1": [ 652, 200 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 640, 17 ], "axis1": [ 652, 200 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 860, 2 ], "axis1": [ 652, 200 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 652, 200 ], "axis1": [ 290, 14 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 652, 200 ], "axis1": [ 305, 2 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 652, 200 ], "axis1": [ 393, 1 ], "matrix": "L15.mlp.W_out" } ], "l:417": [ { "axis0": [ 0, 1 ], "axis1": [ 860, 2 ], "matrix": "L14.mlp.W_out" } ], "l:418": [ { "axis0": [ 0, 1 ], "axis1": [ 505, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 505, 17 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 40, 1 ], "axis1": [ 505, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 64, 1 ], "axis1": [ 505, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 308, 11 ], "axis1": [ 505, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 320, 3 ], "axis1": [ 505, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 328, 1 ], "axis1": [ 505, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 338, 1 ], "axis1": [ 505, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 40, 1 ], "axis1": [ 505, 17 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 64, 1 ], "axis1": [ 505, 17 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 308, 11 ], "axis1": [ 505, 17 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 320, 3 ], "axis1": [ 505, 17 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 328, 1 ], "axis1": [ 505, 17 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 338, 1 ], "axis1": [ 505, 17 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 505, 17 ], "axis1": [ 22, 2 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 505, 17 ], "axis1": [ 106, 5 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 505, 17 ], "axis1": [ 120, 1 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 505, 17 ], "axis1": [ 347, 1 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 505, 17 ], "axis1": [ 395, 1 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 505, 17 ], "axis1": [ 532, 2 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 505, 17 ], "axis1": [ 535, 3 ], "matrix": "L16.mlp.W_out" }, { "axis0": [ 505, 17 ], "axis1": [ 540, 2 ], "matrix": "L16.mlp.W_out" } ], "l:420": [ { "axis0": [ 0, 1 ], "axis1": [ 852, 200 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 852, 200 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 308, 11 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 320, 3 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 328, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 338, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 177, 2 ], "axis1": [ 852, 200 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 180, 9 ], "axis1": [ 852, 200 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 210, 1 ], "axis1": [ 852, 200 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 214, 1 ], "axis1": [ 852, 200 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 217, 1 ], "axis1": [ 852, 200 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 333, 1 ], "axis1": [ 852, 200 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 392, 1 ], "axis1": [ 852, 200 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 404, 1 ], "axis1": [ 852, 200 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 843, 17 ], "axis1": [ 852, 200 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 862, 2 ], "axis1": [ 852, 200 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 852, 200 ], "axis1": [ 64, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 852, 200 ], "axis1": [ 308, 11 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 852, 200 ], "axis1": [ 320, 3 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 852, 200 ], "axis1": [ 328, 1 ], "matrix": "L15.mlp.W_out" }, { "axis0": [ 852, 200 ], "axis1": [ 338, 1 ], "matrix": "L15.mlp.W_out" } ], "l:422": [ { "axis0": [ 0, 1 ], "axis1": [ 862, 2 ], "matrix": "L14.mlp.W_out" } ], "l:423": [ { "axis0": [ 0, 1 ], "axis1": [ 28, 1 ], "matrix": "L25.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 134, 3 ], "matrix": "L25.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 139, 13 ], "matrix": "L25.mlp.W_out" } ], "l:424": [ { "axis0": [ 0, 1 ], "axis1": [ 18, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 18, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 69, 2 ], "axis1": [ 18, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 79, 1 ], "axis1": [ 18, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 94, 8 ], "axis1": [ 18, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 122, 6 ], "axis1": [ 18, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 286, 1 ], "axis1": [ 18, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 69, 2 ], "axis1": [ 18, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 79, 1 ], "axis1": [ 18, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 94, 8 ], "axis1": [ 18, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 122, 6 ], "axis1": [ 18, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 286, 1 ], "axis1": [ 18, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 18, 17 ], "axis1": [ 57, 1 ], "matrix": "L27.mlp.W_out" }, { "axis0": [ 18, 17 ], "axis1": [ 77, 2 ], "matrix": "L27.mlp.W_out" }, { "axis0": [ 18, 17 ], "axis1": [ 281, 4 ], "matrix": "L27.mlp.W_out" }, { "axis0": [ 18, 17 ], "axis1": [ 288, 2 ], "matrix": "L27.mlp.W_out" }, { "axis0": [ 18, 17 ], "axis1": [ 302, 1 ], "matrix": "L27.mlp.W_out" }, { "axis0": [ 18, 17 ], "axis1": [ 337, 1 ], "matrix": "L27.mlp.W_out" }, { "axis0": [ 18, 17 ], "axis1": [ 339, 1 ], "matrix": "L27.mlp.W_out" }, { "axis0": [ 18, 17 ], "axis1": [ 1909, 5 ], "matrix": "L27.mlp.W_out" } ], "l:426": [ { "axis0": [ 0, 1 ], "axis1": [ 218, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 218, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 286, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 36, 2 ], "axis1": [ 218, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 218, 2 ], "axis1": [ 286, 1 ], "matrix": "L4.mlp.W_out" } ], "l:428": [ { "axis0": [ 0, 1 ], "axis1": [ 7, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 7, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 7, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 36, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 24, 1 ], "axis1": [ 7, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 7, 2 ], "axis1": [ 36, 1 ], "matrix": "L3.mlp.W_out" } ], "l:429": [ { "axis0": [ 0, 1 ], "axis1": [ 9, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 9, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 9, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 37, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 24, 1 ], "axis1": [ 9, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 9, 2 ], "axis1": [ 37, 1 ], "matrix": "L3.mlp.W_out" } ], "l:43": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 2 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 25, 1 ], "matrix": "L2.mlp.W_out" }, { "axis0": [ 27, 3 ], "axis1": [ 1, 2 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 1, 2 ], "axis1": [ 25, 1 ], "matrix": "L2.mlp.W_out" } ], "l:430": [ { "axis0": [ 0, 1 ], "axis1": [ 256, 64 ], "matrix": "L27.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 256, 64 ], "matrix": "L27.attn.W_K" }, { "axis0": [ 19, 7 ], "axis1": [ 256, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 102, 19 ], "axis1": [ 256, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 345, 6 ], "axis1": [ 256, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 352, 1 ], "axis1": [ 256, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 395, 1 ], "axis1": [ 256, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 466, 13 ], "axis1": [ 256, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 480, 9 ], "axis1": [ 256, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 490, 8 ], "axis1": [ 256, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 256, 64 ], "axis1": [ 69, 2 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 79, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 94, 8 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 122, 6 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 320, 64 ], "matrix": "L27.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 320, 64 ], "matrix": "L27.attn.W_K" }, { "axis0": [ 498, 4 ], "axis1": [ 320, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 505, 6 ], "axis1": [ 320, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 512, 10 ], "axis1": [ 320, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 525, 9 ], "axis1": [ 320, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 535, 3 ], "axis1": [ 320, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 540, 3 ], "axis1": [ 320, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 544, 5 ], "axis1": [ 320, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 550, 3 ], "axis1": [ 320, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 555, 1 ], "axis1": [ 320, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 557, 2 ], "axis1": [ 320, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 560, 3 ], "axis1": [ 320, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 565, 4 ], "axis1": [ 320, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 570, 4 ], "axis1": [ 320, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 576, 3 ], "axis1": [ 320, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 580, 4 ], "axis1": [ 320, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 320, 64 ], "axis1": [ 69, 2 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 79, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 94, 8 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 320, 64 ], "axis1": [ 122, 6 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 384, 64 ], "matrix": "L27.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 384, 64 ], "matrix": "L27.attn.W_K" }, { "axis0": [ 585, 3 ], "axis1": [ 384, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 601, 30 ], "axis1": [ 384, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 632, 31 ], "axis1": [ 384, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 384, 64 ], "axis1": [ 69, 2 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 79, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 94, 8 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 122, 6 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 448, 64 ], "matrix": "L27.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 448, 64 ], "matrix": "L27.attn.W_K" }, { "axis0": [ 663, 6 ], "axis1": [ 448, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 670, 6 ], "axis1": [ 448, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 448, 64 ], "axis1": [ 69, 2 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 79, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 94, 8 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 122, 6 ], "matrix": "L27.attn.W_O" } ], "l:431": [ { "axis0": [ 0, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 18, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 18, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 27, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 29, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 29, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 30, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 31, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 32, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 33, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 34, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 35, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 36, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 37, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 38, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 40, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 42, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 42, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 43, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 57, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 57, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 65, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 65, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 69, 16 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 122, 12 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 156, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 156, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 157, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 158, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 159, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 160, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 161, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 162, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 163, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 164, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 165, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 166, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 167, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 168, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 169, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 170, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 171, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 172, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 173, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 174, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 175, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 176, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 177, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 178, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 179, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 180, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 181, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 182, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 183, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 184, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 185, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 186, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 187, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 188, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 190, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 190, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 191, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 192, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 193, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 197, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 197, 11 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 207, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 208, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 209, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 210, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 211, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 212, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 213, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 214, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 215, 6 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 222, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 274, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 274, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 275, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 276, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 277, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 278, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 279, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 280, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 281, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 282, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 283, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 284, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 288, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 288, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 289, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 302, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 302, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 326, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 326, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 327, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 329, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 329, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 333, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 333, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 334, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 337, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 337, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 339, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 339, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 392, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 392, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 404, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 404, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 18, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 18, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 27, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 29, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 29, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 30, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 31, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 32, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 33, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 34, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 35, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 36, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 37, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 38, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 40, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 42, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 42, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 43, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 57, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 57, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 65, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 65, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 69, 16 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 122, 12 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 156, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 156, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 157, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 158, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 159, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 160, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 161, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 162, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 163, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 164, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 165, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 166, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 167, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 168, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 169, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 170, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 171, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 172, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 173, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 174, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 175, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 176, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 177, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 178, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 179, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 180, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 181, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 182, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 183, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 184, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 185, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 186, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 187, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 188, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 190, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 190, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 191, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 192, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 193, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 197, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 197, 11 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 207, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 208, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 209, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 210, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 211, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 212, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 213, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 214, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 215, 6 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 222, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 274, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 274, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 275, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 276, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 277, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 278, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 279, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 280, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 281, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 282, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 283, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 284, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 288, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 288, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 289, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 302, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 302, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 326, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 326, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 327, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 329, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 329, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 333, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 333, 2 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 334, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 337, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 337, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 339, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 339, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 392, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 392, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 404, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 404, 1 ], "axis1": [ 205, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 205, 204 ], "axis1": [ 19, 7 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 205, 204 ], "axis1": [ 102, 19 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 205, 204 ], "axis1": [ 345, 6 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 205, 204 ], "axis1": [ 352, 1 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 205, 204 ], "axis1": [ 395, 1 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 205, 204 ], "axis1": [ 466, 13 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 205, 204 ], "axis1": [ 480, 9 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 205, 204 ], "axis1": [ 490, 12 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 205, 204 ], "axis1": [ 505, 6 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 205, 204 ], "axis1": [ 512, 10 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 205, 204 ], "axis1": [ 525, 9 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 205, 204 ], "axis1": [ 535, 3 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 205, 204 ], "axis1": [ 540, 3 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 205, 204 ], "axis1": [ 544, 5 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 205, 204 ], "axis1": [ 550, 3 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 205, 204 ], "axis1": [ 555, 1 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 205, 204 ], "axis1": [ 557, 2 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 205, 204 ], "axis1": [ 560, 3 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 205, 204 ], "axis1": [ 565, 4 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 205, 204 ], "axis1": [ 570, 4 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 205, 204 ], "axis1": [ 576, 3 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 205, 204 ], "axis1": [ 580, 4 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 205, 204 ], "axis1": [ 585, 3 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 205, 204 ], "axis1": [ 601, 30 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 205, 204 ], "axis1": [ 632, 37 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 205, 204 ], "axis1": [ 670, 6 ], "matrix": "L26.mlp.W_out" } ], "l:434": [ { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L5.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L5.attn.W_K" }, { "axis0": [ 173, 51 ], "axis1": [ 64, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 27, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 69, 16 ], "matrix": "L5.attn.W_O" } ], "l:436": [ { "axis0": [ 0, 1 ], "axis1": [ 94, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 94, 17 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 33, 1 ], "axis1": [ 94, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 86, 17 ], "axis1": [ 94, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 33, 1 ], "axis1": [ 94, 17 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 86, 17 ], "axis1": [ 94, 17 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 94, 17 ], "axis1": [ 173, 17 ], "matrix": "L4.mlp.W_out" } ], "l:438": [ { "axis0": [ 0, 1 ], "axis1": [ 86, 17 ], "matrix": "L3.mlp.W_out" } ], "l:439": [ { "axis0": [ 0, 1 ], "axis1": [ 111, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 111, 17 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 35, 1 ], "axis1": [ 111, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 103, 17 ], "axis1": [ 111, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 35, 1 ], "axis1": [ 111, 17 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 103, 17 ], "axis1": [ 111, 17 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 111, 17 ], "axis1": [ 190, 17 ], "matrix": "L4.mlp.W_out" } ], "l:441": [ { "axis0": [ 0, 1 ], "axis1": [ 103, 17 ], "matrix": "L3.mlp.W_out" } ], "l:442": [ { "axis0": [ 0, 1 ], "axis1": [ 128, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 17 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 40, 1 ], "axis1": [ 128, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 120, 17 ], "axis1": [ 128, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 40, 1 ], "axis1": [ 128, 17 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 120, 17 ], "axis1": [ 128, 17 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 128, 17 ], "axis1": [ 207, 17 ], "matrix": "L4.mlp.W_out" } ], "l:444": [ { "axis0": [ 0, 1 ], "axis1": [ 120, 17 ], "matrix": "L3.mlp.W_out" } ], "l:445": [ { "axis0": [ 0, 1 ], "axis1": [ 35, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 35, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 27, 2 ], "axis1": [ 35, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 128, 9 ], "axis1": [ 35, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 139, 6 ], "axis1": [ 35, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 231, 1 ], "axis1": [ 35, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 27, 2 ], "axis1": [ 35, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 128, 9 ], "axis1": [ 35, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 139, 6 ], "axis1": [ 35, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 231, 1 ], "axis1": [ 35, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 35, 17 ], "axis1": [ 1914, 17 ], "matrix": "L27.mlp.W_out" } ], "l:447": [ { "axis0": [ 0, 1 ], "axis1": [ 120, 2 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 120, 2 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 231, 1 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 275, 2 ], "axis1": [ 120, 2 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 120, 2 ], "axis1": [ 231, 1 ], "matrix": "L5.mlp.W_out" } ], "l:449": [ { "axis0": [ 0, 1 ], "axis1": [ 196, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 196, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 275, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 24, 1 ], "axis1": [ 196, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 196, 2 ], "axis1": [ 275, 1 ], "matrix": "L4.mlp.W_out" } ], "l:45": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 2 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 18, 1 ], "axis1": [ 1, 2 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 1, 2 ], "axis1": [ 27, 1 ], "matrix": "L1.mlp.W_out" } ], "l:450": [ { "axis0": [ 0, 1 ], "axis1": [ 198, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 198, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 276, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 24, 1 ], "axis1": [ 198, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 198, 2 ], "axis1": [ 276, 1 ], "matrix": "L4.mlp.W_out" } ], "l:451": [ { "axis0": [ 0, 1 ], "axis1": [ 512, 64 ], "matrix": "L27.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 512, 64 ], "matrix": "L27.attn.W_K" }, { "axis0": [ 66, 1 ], "axis1": [ 512, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 137, 2 ], "axis1": [ 512, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 331, 2 ], "axis1": [ 512, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 353, 1 ], "axis1": [ 512, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 355, 1 ], "axis1": [ 512, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 396, 2 ], "axis1": [ 512, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 676, 3 ], "axis1": [ 512, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 680, 6 ], "axis1": [ 512, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 691, 44 ], "axis1": [ 512, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 736, 2 ], "axis1": [ 512, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 512, 64 ], "axis1": [ 27, 2 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 128, 9 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 512, 64 ], "axis1": [ 139, 6 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 576, 64 ], "matrix": "L27.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 576, 64 ], "matrix": "L27.attn.W_K" }, { "axis0": [ 738, 64 ], "axis1": [ 576, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 576, 64 ], "axis1": [ 27, 2 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 576, 64 ], "axis1": [ 128, 9 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 576, 64 ], "axis1": [ 139, 6 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 640, 64 ], "matrix": "L27.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 640, 64 ], "matrix": "L27.attn.W_K" }, { "axis0": [ 802, 2 ], "axis1": [ 640, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 805, 16 ], "axis1": [ 640, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 822, 33 ], "axis1": [ 640, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 864, 13 ], "axis1": [ 640, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 640, 64 ], "axis1": [ 27, 2 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 640, 64 ], "axis1": [ 128, 9 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 640, 64 ], "axis1": [ 139, 6 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 704, 64 ], "matrix": "L27.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 704, 64 ], "matrix": "L27.attn.W_K" }, { "axis0": [ 877, 12 ], "axis1": [ 704, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 704, 64 ], "axis1": [ 27, 2 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 704, 64 ], "axis1": [ 128, 9 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 704, 64 ], "axis1": [ 139, 6 ], "matrix": "L27.attn.W_O" } ], "l:452": [ { "axis0": [ 0, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 18, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 27, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 27, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 29, 5 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 33, 3 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 35, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 36, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 37, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 38, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 42, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 42, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 43, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 57, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 57, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 65, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 69, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 69, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 70, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 71, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 72, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 73, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 74, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 75, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 76, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 77, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 78, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 79, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 80, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 81, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 82, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 83, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 84, 18 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 122, 12 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 156, 9 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 164, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 165, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 166, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 167, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 168, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 169, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 170, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 171, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 172, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 173, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 174, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 175, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 176, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 177, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 178, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 179, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 180, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 181, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 182, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 183, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 184, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 185, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 186, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 187, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 188, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 190, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 190, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 191, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 192, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 193, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 197, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 197, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 207, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 207, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 208, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 209, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 210, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 211, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 212, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 213, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 214, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 215, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 274, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 274, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 275, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 276, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 277, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 278, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 279, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 280, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 281, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 282, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 283, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 284, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 288, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 288, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 289, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 302, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 302, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 326, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 327, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 329, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 333, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 333, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 334, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 337, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 337, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 339, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 339, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 392, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 392, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 404, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 404, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 18, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 27, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 27, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 29, 5 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 33, 3 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 35, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 36, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 37, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 38, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 42, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 42, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 43, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 57, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 57, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 65, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 69, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 69, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 70, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 71, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 72, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 73, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 74, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 75, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 76, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 77, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 78, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 79, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 80, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 81, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 82, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 83, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 84, 18 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 122, 12 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 156, 9 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 164, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 165, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 166, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 167, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 168, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 169, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 170, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 171, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 172, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 173, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 174, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 175, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 176, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 177, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 178, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 179, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 180, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 181, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 182, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 183, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 184, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 185, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 186, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 187, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 188, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 190, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 190, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 191, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 192, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 193, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 197, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 197, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 207, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 207, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 208, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 209, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 210, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 211, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 212, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 213, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 214, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 215, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 274, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 274, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 275, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 276, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 277, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 278, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 279, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 280, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 281, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 282, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 283, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 284, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 288, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 288, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 289, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 302, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 302, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 326, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 327, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 329, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 333, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 333, 2 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 334, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 337, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 337, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 339, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 339, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 392, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 392, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 404, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 404, 1 ], "axis1": [ 409, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 409, 204 ], "axis1": [ 66, 1 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 409, 204 ], "axis1": [ 137, 2 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 409, 204 ], "axis1": [ 331, 2 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 409, 204 ], "axis1": [ 353, 1 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 409, 204 ], "axis1": [ 355, 1 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 409, 204 ], "axis1": [ 396, 2 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 409, 204 ], "axis1": [ 676, 3 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 409, 204 ], "axis1": [ 680, 6 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 409, 204 ], "axis1": [ 691, 44 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 409, 204 ], "axis1": [ 736, 68 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 409, 204 ], "axis1": [ 805, 16 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 409, 204 ], "axis1": [ 822, 33 ], "matrix": "L26.mlp.W_out" }, { "axis0": [ 409, 204 ], "axis1": [ 864, 25 ], "matrix": "L26.mlp.W_out" } ], "l:455": [ { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L5.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L5.attn.W_K" }, { "axis0": [ 224, 51 ], "axis1": [ 128, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 85, 17 ], "matrix": "L5.attn.W_O" } ], "l:457": [ { "axis0": [ 0, 1 ], "axis1": [ 145, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 145, 17 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 33, 1 ], "axis1": [ 145, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 86, 17 ], "axis1": [ 145, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 33, 1 ], "axis1": [ 145, 17 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 86, 17 ], "axis1": [ 145, 17 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 145, 17 ], "axis1": [ 224, 17 ], "matrix": "L4.mlp.W_out" } ], "l:459": [ { "axis0": [ 0, 1 ], "axis1": [ 162, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 162, 17 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 35, 1 ], "axis1": [ 162, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 103, 17 ], "axis1": [ 162, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 35, 1 ], "axis1": [ 162, 17 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 103, 17 ], "axis1": [ 162, 17 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 162, 17 ], "axis1": [ 241, 17 ], "matrix": "L4.mlp.W_out" } ], "l:46": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 1 ], "matrix": "L0.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L0.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 18, 1 ], "matrix": "L0.mlp.W_out" }, { "axis0": [ 1, 17 ], "axis1": [ 1, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 1, 1 ], "axis1": [ 18, 1 ], "matrix": "L0.mlp.W_out" } ], "l:461": [ { "axis0": [ 0, 1 ], "axis1": [ 179, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 179, 17 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 40, 1 ], "axis1": [ 179, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 120, 17 ], "axis1": [ 179, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 40, 1 ], "axis1": [ 179, 17 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 120, 17 ], "axis1": [ 179, 17 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 179, 17 ], "axis1": [ 258, 17 ], "matrix": "L4.mlp.W_out" } ], "l:463": [ { "axis0": [ 0, 1 ], "axis1": [ 52, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 52, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 29, 3 ], "axis1": [ 52, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 71, 1 ], "axis1": [ 52, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 145, 7 ], "axis1": [ 52, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 156, 6 ], "axis1": [ 52, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 232, 1 ], "axis1": [ 52, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 29, 3 ], "axis1": [ 52, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 71, 1 ], "axis1": [ 52, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 145, 7 ], "axis1": [ 52, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 156, 6 ], "axis1": [ 52, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 232, 1 ], "axis1": [ 52, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 52, 17 ], "axis1": [ 1931, 17 ], "matrix": "L27.mlp.W_out" } ], "l:465": [ { "axis0": [ 0, 1 ], "axis1": [ 122, 2 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 122, 2 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 232, 1 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 277, 2 ], "axis1": [ 122, 2 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 122, 2 ], "axis1": [ 232, 1 ], "matrix": "L5.mlp.W_out" } ], "l:467": [ { "axis0": [ 0, 1 ], "axis1": [ 200, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 200, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 277, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 24, 1 ], "axis1": [ 200, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 200, 2 ], "axis1": [ 277, 1 ], "matrix": "L4.mlp.W_out" } ], "l:468": [ { "axis0": [ 0, 1 ], "axis1": [ 202, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 202, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 278, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 24, 1 ], "axis1": [ 202, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 202, 2 ], "axis1": [ 278, 1 ], "matrix": "L4.mlp.W_out" } ], "l:469": [ { "axis0": [ 0, 1 ], "axis1": [ 768, 64 ], "matrix": "L27.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 768, 64 ], "matrix": "L27.attn.W_K" }, { "axis0": [ 889, 64 ], "axis1": [ 768, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 768, 64 ], "axis1": [ 29, 3 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 71, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 145, 7 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 768, 64 ], "axis1": [ 156, 6 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 832, 64 ], "matrix": "L27.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 832, 64 ], "matrix": "L27.attn.W_K" }, { "axis0": [ 953, 64 ], "axis1": [ 832, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 832, 64 ], "axis1": [ 29, 3 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 71, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 145, 7 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 832, 64 ], "axis1": [ 156, 6 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 896, 64 ], "matrix": "L27.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 896, 64 ], "matrix": "L27.attn.W_K" }, { "axis0": [ 1017, 64 ], "axis1": [ 896, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 896, 64 ], "axis1": [ 29, 3 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 896, 64 ], "axis1": [ 71, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 896, 64 ], "axis1": [ 145, 7 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 896, 64 ], "axis1": [ 156, 6 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 960, 64 ], "matrix": "L27.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 960, 64 ], "matrix": "L27.attn.W_K" }, { "axis0": [ 1081, 12 ], "axis1": [ 960, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 960, 64 ], "axis1": [ 29, 3 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 960, 64 ], "axis1": [ 71, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 960, 64 ], "axis1": [ 145, 7 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 960, 64 ], "axis1": [ 156, 6 ], "matrix": "L27.attn.W_O" } ], "l:47": [ { "axis0": [ 0, 1 ], "axis1": [ 3, 2 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 3, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 19, 1 ], "axis1": [ 3, 2 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 3, 2 ], "axis1": [ 28, 1 ], "matrix": "L1.mlp.W_out" } ], "l:470": [ { "axis0": [ 0, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 27, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 27, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 33, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 33, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 35, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 35, 4 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 42, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 42, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 43, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 57, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 57, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 69, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 69, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 70, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 71, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 72, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 73, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 74, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 75, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 76, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 77, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 78, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 79, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 80, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 81, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 82, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 83, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 84, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 85, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 85, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 86, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 86, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 87, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 87, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 88, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 88, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 89, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 89, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 90, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 90, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 91, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 91, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 92, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 92, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 93, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 93, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 94, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 94, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 95, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 95, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 96, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 96, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 97, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 97, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 98, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 98, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 99, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 99, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 100, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 100, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 101, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 101, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 122, 12 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 164, 10 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 173, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 174, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 175, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 176, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 177, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 178, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 179, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 180, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 181, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 182, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 183, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 184, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 185, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 186, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 187, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 188, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 190, 4 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 197, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 207, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 207, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 208, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 209, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 210, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 211, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 212, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 213, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 214, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 215, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 274, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 274, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 275, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 276, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 277, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 278, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 279, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 280, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 281, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 282, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 283, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 284, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 288, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 288, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 289, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 302, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 302, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 327, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 327, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 333, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 333, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 334, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 337, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 337, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 339, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 339, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 392, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 392, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 404, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 404, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 27, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 27, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 33, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 33, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 35, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 35, 4 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 42, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 42, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 43, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 57, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 57, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 69, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 69, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 70, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 71, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 72, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 73, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 74, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 75, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 76, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 77, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 78, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 79, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 80, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 81, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 82, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 83, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 84, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 85, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 85, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 86, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 86, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 87, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 87, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 88, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 88, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 89, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 89, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 90, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 90, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 91, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 91, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 92, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 92, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 93, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 93, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 94, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 94, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 95, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 95, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 96, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 96, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 97, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 97, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 98, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 98, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 99, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 99, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 100, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 100, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 101, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 101, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 122, 12 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 164, 10 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 173, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 174, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 175, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 176, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 177, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 178, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 179, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 180, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 181, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 182, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 183, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 184, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 185, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 186, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 187, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 188, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 190, 4 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 197, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 207, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 207, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 208, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 209, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 210, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 211, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 212, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 213, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 214, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 215, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 274, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 274, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 275, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 276, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 277, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 278, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 279, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 280, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 281, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 282, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 283, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 284, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 288, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 288, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 289, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 302, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 302, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 327, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 327, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 333, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 333, 2 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 334, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 337, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 337, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 339, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 339, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 392, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 392, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 404, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 404, 1 ], "axis1": [ 613, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 613, 204 ], "axis1": [ 889, 204 ], "matrix": "L26.mlp.W_out" } ], "l:473": [ { "axis0": [ 0, 1 ], "axis1": [ 69, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 69, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 32, 3 ], "axis1": [ 69, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 162, 14 ], "axis1": [ 69, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 233, 1 ], "axis1": [ 69, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 32, 3 ], "axis1": [ 69, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 162, 14 ], "axis1": [ 69, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 233, 1 ], "axis1": [ 69, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 69, 17 ], "axis1": [ 1948, 17 ], "matrix": "L27.mlp.W_out" } ], "l:475": [ { "axis0": [ 0, 1 ], "axis1": [ 124, 2 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 124, 2 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 233, 1 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 279, 2 ], "axis1": [ 124, 2 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 124, 2 ], "axis1": [ 233, 1 ], "matrix": "L5.mlp.W_out" } ], "l:477": [ { "axis0": [ 0, 1 ], "axis1": [ 204, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 204, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 279, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 24, 1 ], "axis1": [ 204, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 204, 2 ], "axis1": [ 279, 1 ], "matrix": "L4.mlp.W_out" } ], "l:478": [ { "axis0": [ 0, 1 ], "axis1": [ 206, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 206, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 280, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 24, 1 ], "axis1": [ 206, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 206, 2 ], "axis1": [ 280, 1 ], "matrix": "L4.mlp.W_out" } ], "l:479": [ { "axis0": [ 0, 1 ], "axis1": [ 1024, 64 ], "matrix": "L27.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1024, 64 ], "matrix": "L27.attn.W_K" }, { "axis0": [ 1093, 64 ], "axis1": [ 1024, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 1024, 64 ], "axis1": [ 32, 3 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1024, 64 ], "axis1": [ 162, 14 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1088, 64 ], "matrix": "L27.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1088, 64 ], "matrix": "L27.attn.W_K" }, { "axis0": [ 1157, 64 ], "axis1": [ 1088, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 1088, 64 ], "axis1": [ 32, 3 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1088, 64 ], "axis1": [ 162, 14 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1152, 64 ], "matrix": "L27.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1152, 64 ], "matrix": "L27.attn.W_K" }, { "axis0": [ 1221, 64 ], "axis1": [ 1152, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 1152, 64 ], "axis1": [ 32, 3 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1152, 64 ], "axis1": [ 162, 14 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1216, 64 ], "matrix": "L27.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1216, 64 ], "matrix": "L27.attn.W_K" }, { "axis0": [ 1285, 12 ], "axis1": [ 1216, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 1216, 64 ], "axis1": [ 32, 3 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1216, 64 ], "axis1": [ 162, 14 ], "matrix": "L27.attn.W_O" } ], "l:48": [ { "axis0": [ 0, 1 ], "axis1": [ 2, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 2, 1 ], "matrix": "L0.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 19, 1 ], "matrix": "L0.mlp.W_out" }, { "axis0": [ 1, 17 ], "axis1": [ 2, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 2, 1 ], "axis1": [ 19, 1 ], "matrix": "L0.mlp.W_out" } ], "l:480": [ { "axis0": [ 0, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 27, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 27, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 33, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 33, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 35, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 42, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 57, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 57, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 69, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 69, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 70, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 71, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 72, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 73, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 74, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 75, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 76, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 77, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 78, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 79, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 80, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 81, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 82, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 83, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 84, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 85, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 85, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 85, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 85, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 86, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 86, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 86, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 86, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 87, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 87, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 87, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 87, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 88, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 88, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 88, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 88, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 89, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 89, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 89, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 89, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 90, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 90, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 90, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 90, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 91, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 91, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 91, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 91, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 92, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 92, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 92, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 92, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 93, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 93, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 93, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 93, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 94, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 94, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 94, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 94, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 95, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 95, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 95, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 95, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 96, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 96, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 96, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 96, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 97, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 97, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 97, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 97, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 98, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 98, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 98, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 98, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 99, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 99, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 99, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 99, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 100, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 100, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 100, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 100, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 101, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 101, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 101, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 101, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 122, 12 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 173, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 173, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 174, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 175, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 176, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 177, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 178, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 179, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 180, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 181, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 182, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 183, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 184, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 185, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 186, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 187, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 207, 9 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 274, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 274, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 275, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 276, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 277, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 278, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 279, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 280, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 281, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 282, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 283, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 284, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 288, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 288, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 289, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 302, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 302, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 327, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 333, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 337, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 337, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 339, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 339, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 392, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 392, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 404, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 27, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 27, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 33, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 33, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 35, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 42, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 57, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 57, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 69, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 69, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 70, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 71, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 72, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 73, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 74, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 75, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 76, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 77, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 78, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 79, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 80, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 81, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 82, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 83, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 84, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 85, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 85, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 85, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 85, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 86, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 86, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 86, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 86, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 87, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 87, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 87, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 87, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 88, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 88, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 88, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 88, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 89, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 89, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 89, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 89, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 90, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 90, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 90, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 90, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 91, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 91, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 91, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 91, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 92, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 92, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 92, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 92, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 93, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 93, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 93, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 93, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 94, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 94, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 94, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 94, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 95, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 95, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 95, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 95, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 96, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 96, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 96, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 96, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 97, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 97, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 97, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 97, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 98, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 98, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 98, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 98, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 99, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 99, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 99, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 99, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 100, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 100, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 100, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 100, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 101, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 101, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 101, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 101, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 122, 12 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 173, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 173, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 174, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 175, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 176, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 177, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 178, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 179, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 180, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 181, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 182, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 183, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 184, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 185, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 186, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 187, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 207, 9 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 274, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 274, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 275, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 276, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 277, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 278, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 279, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 280, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 281, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 282, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 283, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 284, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 288, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 288, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 289, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 302, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 302, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 327, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 333, 2 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 337, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 337, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 339, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 339, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 392, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 392, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 404, 1 ], "axis1": [ 817, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 817, 204 ], "axis1": [ 1093, 204 ], "matrix": "L26.mlp.W_out" } ], "l:483": [ { "axis0": [ 0, 1 ], "axis1": [ 86, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 86, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 35, 3 ], "axis1": [ 86, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 72, 1 ], "axis1": [ 86, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 176, 12 ], "axis1": [ 86, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 287, 1 ], "axis1": [ 86, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 392, 1 ], "axis1": [ 86, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 35, 3 ], "axis1": [ 86, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 72, 1 ], "axis1": [ 86, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 176, 12 ], "axis1": [ 86, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 287, 1 ], "axis1": [ 86, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 392, 1 ], "axis1": [ 86, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 86, 17 ], "axis1": [ 1965, 17 ], "matrix": "L27.mlp.W_out" } ], "l:485": [ { "axis0": [ 0, 1 ], "axis1": [ 220, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 220, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 287, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 38, 1 ], "axis1": [ 220, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 46, 1 ], "axis1": [ 220, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 220, 2 ], "axis1": [ 287, 1 ], "matrix": "L4.mlp.W_out" } ], "l:487": [ { "axis0": [ 0, 1 ], "axis1": [ 11, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 11, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 11, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 38, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 24, 1 ], "axis1": [ 11, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 11, 2 ], "axis1": [ 38, 1 ], "matrix": "L3.mlp.W_out" } ], "l:488": [ { "axis0": [ 0, 1 ], "axis1": [ 13, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 13, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 13, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 46, 1 ], "matrix": "L3.mlp.W_out" }, { "axis0": [ 24, 1 ], "axis1": [ 13, 2 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 13, 2 ], "axis1": [ 46, 1 ], "matrix": "L3.mlp.W_out" } ], "l:489": [ { "axis0": [ 0, 1 ], "axis1": [ 1280, 64 ], "matrix": "L27.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1280, 64 ], "matrix": "L27.attn.W_K" }, { "axis0": [ 1297, 64 ], "axis1": [ 1280, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 1280, 64 ], "axis1": [ 35, 3 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1280, 64 ], "axis1": [ 72, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1280, 64 ], "axis1": [ 176, 12 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1280, 64 ], "axis1": [ 392, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1344, 64 ], "matrix": "L27.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1344, 64 ], "matrix": "L27.attn.W_K" }, { "axis0": [ 1361, 64 ], "axis1": [ 1344, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 1344, 64 ], "axis1": [ 35, 3 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1344, 64 ], "axis1": [ 72, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1344, 64 ], "axis1": [ 176, 12 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1344, 64 ], "axis1": [ 392, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1408, 64 ], "matrix": "L27.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1408, 64 ], "matrix": "L27.attn.W_K" }, { "axis0": [ 1425, 64 ], "axis1": [ 1408, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 1408, 64 ], "axis1": [ 35, 3 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1408, 64 ], "axis1": [ 72, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1408, 64 ], "axis1": [ 176, 12 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1408, 64 ], "axis1": [ 392, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1472, 64 ], "matrix": "L27.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1472, 64 ], "matrix": "L27.attn.W_K" }, { "axis0": [ 1489, 12 ], "axis1": [ 1472, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 1472, 64 ], "axis1": [ 35, 3 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1472, 64 ], "axis1": [ 72, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1472, 64 ], "axis1": [ 176, 12 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1472, 64 ], "axis1": [ 392, 1 ], "matrix": "L27.attn.W_O" } ], "l:49": [ { "axis0": [ 0, 1 ], "axis1": [ 5, 2 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 5, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 20, 1 ], "axis1": [ 5, 2 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 5, 2 ], "axis1": [ 29, 1 ], "matrix": "L1.mlp.W_out" } ], "l:490": [ { "axis0": [ 0, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 27, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 27, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 33, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 33, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 57, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 69, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 69, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 70, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 71, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 72, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 73, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 74, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 75, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 76, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 77, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 78, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 79, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 80, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 81, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 82, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 83, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 84, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 85, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 85, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 85, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 85, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 85, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 85, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 86, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 86, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 86, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 86, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 86, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 86, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 87, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 87, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 87, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 87, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 87, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 87, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 88, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 88, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 88, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 88, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 88, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 88, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 89, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 89, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 89, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 89, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 89, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 89, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 90, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 90, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 90, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 90, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 90, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 90, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 91, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 91, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 91, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 91, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 91, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 91, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 92, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 92, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 92, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 92, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 92, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 92, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 93, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 93, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 93, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 93, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 93, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 93, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 94, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 94, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 94, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 94, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 94, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 94, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 95, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 95, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 95, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 95, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 95, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 95, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 96, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 96, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 96, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 96, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 96, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 96, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 97, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 97, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 97, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 97, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 97, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 97, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 98, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 98, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 98, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 98, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 98, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 98, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 99, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 99, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 99, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 99, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 99, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 99, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 100, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 100, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 100, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 100, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 100, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 100, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 101, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 101, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 101, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 101, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 101, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 101, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 122, 12 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 173, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 173, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 174, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 175, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 176, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 177, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 178, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 179, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 180, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 181, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 182, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 183, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 184, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 185, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 186, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 187, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 274, 11 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 288, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 302, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 337, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 339, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 392, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 392, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 27, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 27, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 33, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 33, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 57, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 69, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 69, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 70, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 71, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 72, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 73, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 74, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 75, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 76, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 77, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 78, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 79, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 80, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 81, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 82, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 83, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 84, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 85, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 85, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 85, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 85, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 85, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 85, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 86, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 86, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 86, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 86, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 86, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 86, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 87, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 87, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 87, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 87, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 87, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 87, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 88, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 88, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 88, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 88, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 88, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 88, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 89, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 89, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 89, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 89, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 89, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 89, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 90, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 90, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 90, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 90, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 90, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 90, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 91, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 91, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 91, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 91, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 91, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 91, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 92, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 92, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 92, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 92, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 92, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 92, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 93, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 93, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 93, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 93, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 93, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 93, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 94, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 94, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 94, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 94, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 94, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 94, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 95, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 95, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 95, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 95, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 95, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 95, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 96, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 96, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 96, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 96, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 96, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 96, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 97, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 97, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 97, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 97, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 97, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 97, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 98, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 98, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 98, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 98, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 98, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 98, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 99, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 99, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 99, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 99, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 99, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 99, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 100, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 100, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 100, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 100, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 100, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 100, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 101, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 101, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 101, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 101, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 101, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 101, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 122, 12 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 173, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 173, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 174, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 175, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 176, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 177, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 178, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 179, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 180, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 181, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 182, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 183, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 184, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 185, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 186, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 187, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 274, 11 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 288, 2 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 302, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 337, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 339, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 392, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 392, 1 ], "axis1": [ 1021, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 1021, 204 ], "axis1": [ 1297, 204 ], "matrix": "L26.mlp.W_out" } ], "l:493": [ { "axis0": [ 0, 1 ], "axis1": [ 103, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 103, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 38, 1 ], "axis1": [ 103, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 40, 1 ], "axis1": [ 103, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 73, 1 ], "axis1": [ 103, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 188, 1 ], "axis1": [ 103, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 190, 4 ], "axis1": [ 103, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 197, 6 ], "axis1": [ 103, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 234, 1 ], "axis1": [ 103, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 327, 1 ], "axis1": [ 103, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 333, 1 ], "axis1": [ 103, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 404, 1 ], "axis1": [ 103, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 38, 1 ], "axis1": [ 103, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 40, 1 ], "axis1": [ 103, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 73, 1 ], "axis1": [ 103, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 188, 1 ], "axis1": [ 103, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 190, 4 ], "axis1": [ 103, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 197, 6 ], "axis1": [ 103, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 234, 1 ], "axis1": [ 103, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 327, 1 ], "axis1": [ 103, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 333, 1 ], "axis1": [ 103, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 404, 1 ], "axis1": [ 103, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 103, 17 ], "axis1": [ 1982, 17 ], "matrix": "L27.mlp.W_out" } ], "l:495": [ { "axis0": [ 0, 1 ], "axis1": [ 126, 2 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 126, 2 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 234, 1 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 281, 2 ], "axis1": [ 126, 2 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 126, 2 ], "axis1": [ 234, 1 ], "matrix": "L5.mlp.W_out" } ], "l:497": [ { "axis0": [ 0, 1 ], "axis1": [ 208, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 208, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 281, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 24, 1 ], "axis1": [ 208, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 208, 2 ], "axis1": [ 281, 1 ], "matrix": "L4.mlp.W_out" } ], "l:498": [ { "axis0": [ 0, 1 ], "axis1": [ 210, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 210, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 282, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 24, 1 ], "axis1": [ 210, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 210, 2 ], "axis1": [ 282, 1 ], "matrix": "L4.mlp.W_out" } ], "l:499": [ { "axis0": [ 0, 1 ], "axis1": [ 1536, 64 ], "matrix": "L27.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1536, 64 ], "matrix": "L27.attn.W_K" }, { "axis0": [ 1501, 64 ], "axis1": [ 1536, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 1536, 64 ], "axis1": [ 38, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1536, 64 ], "axis1": [ 40, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1536, 64 ], "axis1": [ 73, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1536, 64 ], "axis1": [ 188, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1536, 64 ], "axis1": [ 190, 4 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1536, 64 ], "axis1": [ 197, 6 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1536, 64 ], "axis1": [ 327, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1536, 64 ], "axis1": [ 333, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1536, 64 ], "axis1": [ 404, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1600, 64 ], "matrix": "L27.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1600, 64 ], "matrix": "L27.attn.W_K" }, { "axis0": [ 1565, 64 ], "axis1": [ 1600, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 1600, 64 ], "axis1": [ 38, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1600, 64 ], "axis1": [ 40, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1600, 64 ], "axis1": [ 73, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1600, 64 ], "axis1": [ 188, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1600, 64 ], "axis1": [ 190, 4 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1600, 64 ], "axis1": [ 197, 6 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1600, 64 ], "axis1": [ 327, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1600, 64 ], "axis1": [ 333, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1600, 64 ], "axis1": [ 404, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1664, 64 ], "matrix": "L27.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1664, 64 ], "matrix": "L27.attn.W_K" }, { "axis0": [ 1629, 64 ], "axis1": [ 1664, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 1664, 64 ], "axis1": [ 38, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1664, 64 ], "axis1": [ 40, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1664, 64 ], "axis1": [ 73, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1664, 64 ], "axis1": [ 188, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1664, 64 ], "axis1": [ 190, 4 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1664, 64 ], "axis1": [ 197, 6 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1664, 64 ], "axis1": [ 327, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1664, 64 ], "axis1": [ 333, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1664, 64 ], "axis1": [ 404, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1728, 64 ], "matrix": "L27.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1728, 64 ], "matrix": "L27.attn.W_K" }, { "axis0": [ 1693, 12 ], "axis1": [ 1728, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 1728, 64 ], "axis1": [ 38, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1728, 64 ], "axis1": [ 40, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1728, 64 ], "axis1": [ 73, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1728, 64 ], "axis1": [ 188, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1728, 64 ], "axis1": [ 190, 4 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1728, 64 ], "axis1": [ 197, 6 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1728, 64 ], "axis1": [ 327, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1728, 64 ], "axis1": [ 333, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1728, 64 ], "axis1": [ 404, 1 ], "matrix": "L27.attn.W_O" } ], "l:5": [ { "axis0": [ 0, 1 ], "axis1": [ 40, 1 ], "matrix": "L0.mlp.W_out" } ], "l:50": [ { "axis0": [ 0, 1 ], "axis1": [ 3, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 3, 1 ], "matrix": "L0.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 20, 1 ], "matrix": "L0.mlp.W_out" }, { "axis0": [ 1, 17 ], "axis1": [ 3, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 3, 1 ], "axis1": [ 20, 1 ], "matrix": "L0.mlp.W_out" } ], "l:500": [ { "axis0": [ 0, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 27, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 27, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 33, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 69, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 69, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 70, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 71, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 72, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 73, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 74, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 75, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 76, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 77, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 78, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 79, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 80, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 81, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 82, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 83, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 84, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 85, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 85, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 85, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 85, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 85, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 85, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 85, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 85, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 86, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 86, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 86, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 86, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 86, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 86, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 86, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 86, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 87, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 87, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 87, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 87, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 87, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 87, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 87, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 87, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 88, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 88, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 88, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 88, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 88, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 88, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 88, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 88, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 89, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 89, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 89, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 89, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 89, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 89, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 89, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 89, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 90, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 90, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 90, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 90, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 90, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 90, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 90, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 90, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 91, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 91, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 91, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 91, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 91, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 91, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 91, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 91, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 92, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 92, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 92, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 92, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 92, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 92, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 92, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 92, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 93, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 93, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 93, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 93, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 93, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 93, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 93, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 93, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 94, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 94, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 94, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 94, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 94, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 94, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 94, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 94, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 95, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 95, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 95, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 95, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 95, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 95, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 95, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 95, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 96, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 96, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 96, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 96, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 96, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 96, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 96, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 96, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 97, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 97, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 97, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 97, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 97, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 97, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 97, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 97, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 98, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 98, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 98, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 98, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 98, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 98, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 98, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 98, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 99, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 99, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 99, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 99, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 99, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 99, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 99, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 99, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 100, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 100, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 100, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 100, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 100, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 100, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 100, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 100, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 101, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 101, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 101, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 101, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 101, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 101, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 101, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 101, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 122, 12 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 173, 15 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 392, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 27, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 27, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 33, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 69, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 69, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 70, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 71, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 72, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 73, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 74, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 75, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 76, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 77, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 78, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 79, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 80, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 81, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 82, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 83, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 84, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 85, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 85, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 85, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 85, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 85, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 85, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 85, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 85, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 86, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 86, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 86, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 86, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 86, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 86, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 86, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 86, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 87, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 87, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 87, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 87, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 87, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 87, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 87, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 87, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 88, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 88, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 88, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 88, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 88, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 88, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 88, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 88, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 89, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 89, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 89, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 89, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 89, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 89, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 89, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 89, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 90, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 90, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 90, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 90, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 90, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 90, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 90, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 90, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 91, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 91, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 91, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 91, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 91, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 91, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 91, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 91, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 92, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 92, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 92, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 92, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 92, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 92, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 92, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 92, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 93, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 93, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 93, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 93, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 93, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 93, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 93, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 93, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 94, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 94, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 94, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 94, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 94, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 94, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 94, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 94, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 95, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 95, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 95, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 95, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 95, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 95, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 95, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 95, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 96, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 96, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 96, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 96, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 96, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 96, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 96, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 96, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 97, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 97, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 97, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 97, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 97, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 97, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 97, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 97, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 98, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 98, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 98, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 98, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 98, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 98, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 98, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 98, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 99, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 99, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 99, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 99, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 99, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 99, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 99, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 99, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 100, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 100, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 100, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 100, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 100, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 100, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 100, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 100, 2 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 101, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 101, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 101, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 101, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 101, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 101, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 101, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 101, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 122, 12 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 173, 15 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 392, 1 ], "axis1": [ 1225, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 1225, 204 ], "axis1": [ 1501, 204 ], "matrix": "L26.mlp.W_out" } ], "l:503": [ { "axis0": [ 0, 1 ], "axis1": [ 120, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 120, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 42, 2 ], "axis1": [ 120, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 203, 14 ], "axis1": [ 120, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 235, 1 ], "axis1": [ 120, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 334, 1 ], "axis1": [ 120, 17 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 42, 2 ], "axis1": [ 120, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 203, 14 ], "axis1": [ 120, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 235, 1 ], "axis1": [ 120, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 334, 1 ], "axis1": [ 120, 17 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 120, 17 ], "axis1": [ 1999, 17 ], "matrix": "L27.mlp.W_out" } ], "l:505": [ { "axis0": [ 0, 1 ], "axis1": [ 128, 2 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 2 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 235, 1 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 283, 2 ], "axis1": [ 128, 2 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 128, 2 ], "axis1": [ 235, 1 ], "matrix": "L5.mlp.W_out" } ], "l:507": [ { "axis0": [ 0, 1 ], "axis1": [ 212, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 212, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 283, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 24, 1 ], "axis1": [ 212, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 212, 2 ], "axis1": [ 283, 1 ], "matrix": "L4.mlp.W_out" } ], "l:508": [ { "axis0": [ 0, 1 ], "axis1": [ 214, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 214, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 284, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 24, 1 ], "axis1": [ 214, 2 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 214, 2 ], "axis1": [ 284, 1 ], "matrix": "L4.mlp.W_out" } ], "l:509": [ { "axis0": [ 0, 1 ], "axis1": [ 1792, 64 ], "matrix": "L27.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1792, 64 ], "matrix": "L27.attn.W_K" }, { "axis0": [ 1705, 64 ], "axis1": [ 1792, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 1792, 64 ], "axis1": [ 42, 2 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1792, 64 ], "axis1": [ 203, 14 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1792, 64 ], "axis1": [ 334, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1856, 64 ], "matrix": "L27.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1856, 64 ], "matrix": "L27.attn.W_K" }, { "axis0": [ 1769, 64 ], "axis1": [ 1856, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 1856, 64 ], "axis1": [ 42, 2 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1856, 64 ], "axis1": [ 203, 14 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1856, 64 ], "axis1": [ 334, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1920, 64 ], "matrix": "L27.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1920, 64 ], "matrix": "L27.attn.W_K" }, { "axis0": [ 1833, 64 ], "axis1": [ 1920, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 1920, 64 ], "axis1": [ 42, 2 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1920, 64 ], "axis1": [ 203, 14 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1920, 64 ], "axis1": [ 334, 1 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 1984, 64 ], "matrix": "L27.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 1984, 64 ], "matrix": "L27.attn.W_K" }, { "axis0": [ 1897, 12 ], "axis1": [ 1984, 64 ], "matrix": "L27.attn.W_V" }, { "axis0": [ 1984, 64 ], "axis1": [ 42, 2 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1984, 64 ], "axis1": [ 203, 14 ], "matrix": "L27.attn.W_O" }, { "axis0": [ 1984, 64 ], "axis1": [ 334, 1 ], "matrix": "L27.attn.W_O" } ], "l:51": [ { "axis0": [ 0, 1 ], "axis1": [ 7, 2 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 7, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 40, 1 ], "matrix": "L1.mlp.W_out" }, { "axis0": [ 24, 1 ], "axis1": [ 7, 2 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 7, 2 ], "axis1": [ 40, 1 ], "matrix": "L1.mlp.W_out" } ], "l:510": [ { "axis0": [ 0, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 27, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 69, 17 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 85, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 85, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 85, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 85, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 85, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 85, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 85, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 85, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 85, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 85, 2 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 86, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 86, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 86, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 86, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 86, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 86, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 86, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 86, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 86, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 86, 2 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 87, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 87, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 87, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 87, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 87, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 87, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 87, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 87, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 87, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 87, 2 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 88, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 88, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 88, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 88, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 88, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 88, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 88, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 88, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 88, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 88, 2 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 89, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 89, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 89, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 89, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 89, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 89, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 89, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 89, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 89, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 89, 2 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 90, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 90, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 90, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 90, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 90, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 90, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 90, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 90, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 90, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 90, 2 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 91, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 91, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 91, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 91, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 91, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 91, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 91, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 91, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 91, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 91, 2 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 92, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 92, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 92, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 92, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 92, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 92, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 92, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 92, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 92, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 92, 2 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 93, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 93, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 93, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 93, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 93, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 93, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 93, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 93, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 93, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 93, 2 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 94, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 94, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 94, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 94, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 94, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 94, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 94, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 94, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 94, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 94, 2 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 95, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 95, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 95, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 95, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 95, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 95, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 95, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 95, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 95, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 95, 2 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 96, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 96, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 96, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 96, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 96, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 96, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 96, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 96, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 96, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 96, 2 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 97, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 97, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 97, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 97, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 97, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 97, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 97, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 97, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 97, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 97, 2 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 98, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 98, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 98, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 98, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 98, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 98, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 98, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 98, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 98, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 98, 2 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 99, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 99, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 99, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 99, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 99, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 99, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 99, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 99, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 99, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 99, 2 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 100, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 100, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 100, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 100, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 100, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 100, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 100, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 100, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 100, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 100, 2 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 101, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 101, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 101, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 101, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 101, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 101, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 101, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 101, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 101, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 101, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 122, 12 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 27, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 69, 17 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 85, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 85, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 85, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 85, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 85, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 85, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 85, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 85, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 85, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 85, 2 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 86, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 86, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 86, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 86, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 86, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 86, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 86, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 86, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 86, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 86, 2 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 87, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 87, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 87, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 87, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 87, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 87, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 87, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 87, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 87, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 87, 2 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 88, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 88, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 88, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 88, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 88, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 88, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 88, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 88, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 88, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 88, 2 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 89, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 89, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 89, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 89, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 89, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 89, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 89, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 89, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 89, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 89, 2 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 90, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 90, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 90, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 90, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 90, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 90, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 90, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 90, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 90, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 90, 2 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 91, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 91, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 91, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 91, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 91, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 91, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 91, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 91, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 91, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 91, 2 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 92, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 92, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 92, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 92, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 92, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 92, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 92, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 92, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 92, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 92, 2 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 93, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 93, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 93, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 93, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 93, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 93, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 93, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 93, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 93, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 93, 2 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 94, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 94, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 94, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 94, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 94, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 94, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 94, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 94, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 94, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 94, 2 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 95, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 95, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 95, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 95, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 95, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 95, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 95, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 95, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 95, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 95, 2 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 96, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 96, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 96, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 96, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 96, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 96, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 96, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 96, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 96, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 96, 2 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 97, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 97, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 97, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 97, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 97, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 97, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 97, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 97, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 97, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 97, 2 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 98, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 98, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 98, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 98, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 98, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 98, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 98, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 98, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 98, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 98, 2 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 99, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 99, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 99, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 99, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 99, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 99, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 99, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 99, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 99, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 99, 2 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 100, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 100, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 100, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 100, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 100, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 100, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 100, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 100, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 100, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 100, 2 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 101, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 101, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 101, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 101, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 101, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 101, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 101, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 101, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 101, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 101, 1 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 122, 12 ], "axis1": [ 1429, 204 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 1429, 204 ], "axis1": [ 1705, 204 ], "matrix": "L26.mlp.W_out" } ], "l:513": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 2016, 17 ], "matrix": "L27.mlp.W_out" } ], "l:52": [ { "axis0": [ 24, 2 ], "axis1": [ 0, 64 ], "matrix": "L1.attn.W_Q" }, { "axis0": [ 21, 1 ], "axis1": [ 0, 64 ], "matrix": "L1.attn.W_K" }, { "axis0": [ 26, 1 ], "axis1": [ 0, 64 ], "matrix": "L1.attn.W_K" }, { "axis0": [ 21, 1 ], "axis1": [ 0, 64 ], "matrix": "L1.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 24, 1 ], "matrix": "L1.attn.W_O" } ], "l:54": [ { "axis0": [ 0, 1 ], "axis1": [ 24, 1 ], "matrix": "L0.mlp.W_out" } ], "l:55": [ { "axis0": [ 0, 1 ], "axis1": [ 25, 1 ], "matrix": "L0.mlp.W_out" } ], "l:57": [ { "axis0": [ 0, 1 ], "axis1": [ 26, 1 ], "matrix": "L0.mlp.W_out" } ], "l:58": [ { "axis0": [ 0, 1 ], "axis1": [ 62, 1 ], "matrix": "L3.mlp.W_out" } ], "l:59": [ { "axis0": [ 0, 1 ], "axis1": [ 69, 17 ], "matrix": "L3.mlp.W_out" } ], "l:60": [ { "axis0": [ 0, 1 ], "axis1": [ 69, 17 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 69, 17 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 35, 1 ], "axis1": [ 69, 17 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 306, 17 ], "axis1": [ 69, 17 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 35, 1 ], "axis1": [ 69, 17 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 306, 17 ], "axis1": [ 69, 17 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 69, 17 ], "axis1": [ 28, 1 ], "matrix": "L5.mlp.W_out" }, { "axis0": [ 69, 17 ], "axis1": [ 121, 16 ], "matrix": "L5.mlp.W_out" } ], "l:62": [ { "axis0": [ 63, 2 ], "axis1": [ 256, 64 ], "matrix": "L4.attn.W_Q" }, { "axis0": [ 27, 1 ], "axis1": [ 256, 64 ], "matrix": "L4.attn.W_K" }, { "axis0": [ 65, 1 ], "axis1": [ 256, 64 ], "matrix": "L4.attn.W_K" }, { "axis0": [ 19, 1 ], "axis1": [ 256, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 256, 64 ], "axis1": [ 35, 1 ], "matrix": "L4.attn.W_O" } ], "l:64": [ { "axis0": [ 0, 1 ], "axis1": [ 63, 1 ], "matrix": "L3.mlp.W_out" } ], "l:65": [ { "axis0": [ 0, 1 ], "axis1": [ 64, 1 ], "matrix": "L3.mlp.W_out" } ], "l:67": [ { "axis0": [ 0, 1 ], "axis1": [ 65, 1 ], "matrix": "L3.mlp.W_out" } ], "l:68": [ { "axis0": [ 0, 1 ], "axis1": [ 306, 17 ], "matrix": "L4.mlp.W_out" } ], "l:69": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 17 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 17 ], "matrix": "L20.mlp.W_up" }, { "axis0": [ 40, 1 ], "axis1": [ 1, 17 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 44, 2 ], "axis1": [ 1, 17 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 51, 1 ], "axis1": [ 1, 17 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 216, 5 ], "axis1": [ 1, 17 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 222, 2 ], "axis1": [ 1, 17 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 236, 4 ], "axis1": [ 1, 17 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 256, 1 ], "axis1": [ 1, 17 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 262, 2 ], "axis1": [ 1, 17 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 40, 1 ], "axis1": [ 1, 17 ], "matrix": "L20.mlp.W_up" }, { "axis0": [ 44, 2 ], "axis1": [ 1, 17 ], "matrix": "L20.mlp.W_up" }, { "axis0": [ 51, 1 ], "axis1": [ 1, 17 ], "matrix": "L20.mlp.W_up" }, { "axis0": [ 216, 5 ], "axis1": [ 1, 17 ], "matrix": "L20.mlp.W_up" }, { "axis0": [ 222, 2 ], "axis1": [ 1, 17 ], "matrix": "L20.mlp.W_up" }, { "axis0": [ 236, 4 ], "axis1": [ 1, 17 ], "matrix": "L20.mlp.W_up" }, { "axis0": [ 256, 1 ], "axis1": [ 1, 17 ], "matrix": "L20.mlp.W_up" }, { "axis0": [ 262, 2 ], "axis1": [ 1, 17 ], "matrix": "L20.mlp.W_up" }, { "axis0": [ 1, 17 ], "axis1": [ 39, 1 ], "matrix": "L20.mlp.W_out" }, { "axis0": [ 1, 17 ], "axis1": [ 49, 1 ], "matrix": "L20.mlp.W_out" }, { "axis0": [ 1, 17 ], "axis1": [ 194, 3 ], "matrix": "L20.mlp.W_out" }, { "axis0": [ 1, 17 ], "axis1": [ 240, 7 ], "matrix": "L20.mlp.W_out" }, { "axis0": [ 1, 17 ], "axis1": [ 248, 3 ], "matrix": "L20.mlp.W_out" }, { "axis0": [ 1, 17 ], "axis1": [ 252, 1 ], "matrix": "L20.mlp.W_out" }, { "axis0": [ 1, 17 ], "axis1": [ 335, 1 ], "matrix": "L20.mlp.W_out" } ], "l:7": [ { "axis0": [ 0, 1 ], "axis1": [ 4, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 4, 1 ], "matrix": "L0.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 21, 1 ], "matrix": "L0.mlp.W_out" }, { "axis0": [ 1, 17 ], "axis1": [ 4, 1 ], "matrix": "L0.mlp.W_in" }, { "axis0": [ 4, 1 ], "axis1": [ 21, 1 ], "matrix": "L0.mlp.W_out" } ], "l:71": [ { "axis0": [ 66, 2 ], "axis1": [ 320, 64 ], "matrix": "L4.attn.W_Q" }, { "axis0": [ 27, 1 ], "axis1": [ 320, 64 ], "matrix": "L4.attn.W_K" }, { "axis0": [ 68, 1 ], "axis1": [ 320, 64 ], "matrix": "L4.attn.W_K" }, { "axis0": [ 20, 1 ], "axis1": [ 320, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 320, 64 ], "axis1": [ 40, 1 ], "matrix": "L4.attn.W_O" } ], "l:73": [ { "axis0": [ 0, 1 ], "axis1": [ 66, 1 ], "matrix": "L3.mlp.W_out" } ], "l:74": [ { "axis0": [ 0, 1 ], "axis1": [ 67, 1 ], "matrix": "L3.mlp.W_out" } ], "l:76": [ { "axis0": [ 0, 1 ], "axis1": [ 68, 1 ], "matrix": "L3.mlp.W_out" } ], "l:77": [ { "axis0": [ 0, 1 ], "axis1": [ 1, 200 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 200 ], "matrix": "L19.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L19.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 44, 2 ], "matrix": "L19.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 51, 1 ], "matrix": "L19.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 216, 5 ], "matrix": "L19.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 222, 2 ], "matrix": "L19.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 236, 4 ], "matrix": "L19.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 256, 1 ], "matrix": "L19.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 262, 2 ], "matrix": "L19.mlp.W_out" }, { "axis0": [ 65, 1 ], "axis1": [ 1, 200 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 326, 1 ], "axis1": [ 1, 200 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 341, 1 ], "axis1": [ 1, 200 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 343, 1 ], "axis1": [ 1, 200 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 377, 12 ], "axis1": [ 1, 200 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 390, 1 ], "axis1": [ 1, 200 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 394, 1 ], "axis1": [ 1, 200 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 398, 1 ], "axis1": [ 1, 200 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 472, 17 ], "axis1": [ 1, 200 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 1, 200 ], "axis1": [ 44, 2 ], "matrix": "L19.mlp.W_out" }, { "axis0": [ 1, 200 ], "axis1": [ 51, 1 ], "matrix": "L19.mlp.W_out" }, { "axis0": [ 1, 200 ], "axis1": [ 216, 5 ], "matrix": "L19.mlp.W_out" }, { "axis0": [ 1, 200 ], "axis1": [ 222, 2 ], "matrix": "L19.mlp.W_out" }, { "axis0": [ 1, 200 ], "axis1": [ 236, 4 ], "matrix": "L19.mlp.W_out" }, { "axis0": [ 1, 200 ], "axis1": [ 256, 1 ], "matrix": "L19.mlp.W_out" }, { "axis0": [ 1, 200 ], "axis1": [ 262, 2 ], "matrix": "L19.mlp.W_out" } ], "l:79": [ { "axis0": [ 0, 1 ], "axis1": [ 1300, 146 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 1300, 146 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 472, 17 ], "matrix": "L13.mlp.W_out" }, { "axis0": [ 357, 1 ], "axis1": [ 1300, 146 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 1300, 146 ], "axis1": [ 472, 17 ], "matrix": "L13.mlp.W_out" } ], "l:80": [ { "axis0": [ 0, 1 ], "axis1": [ 35, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 308, 11 ], "axis1": [ 35, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 320, 3 ], "axis1": [ 35, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 331, 20 ], "axis1": [ 35, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 308, 11 ], "axis1": [ 36, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 320, 3 ], "axis1": [ 36, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 331, 20 ], "axis1": [ 36, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 35, 1 ], "axis1": [ 357, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 36, 1 ], "axis1": [ 357, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" } ], "l:81": [ { "axis0": [ 0, 1 ], "axis1": [ 101, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 101, 100 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 308, 11 ], "matrix": "L11.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 320, 3 ], "matrix": "L11.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 331, 20 ], "matrix": "L11.mlp.W_out" }, { "axis0": [ 49, 2 ], "axis1": [ 101, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 174, 16 ], "axis1": [ 101, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 210, 1 ], "axis1": [ 101, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 239, 8 ], "axis1": [ 101, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 248, 3 ], "axis1": [ 101, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 252, 4 ], "axis1": [ 101, 100 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 101, 100 ], "axis1": [ 308, 11 ], "matrix": "L11.mlp.W_out" }, { "axis0": [ 101, 100 ], "axis1": [ 320, 3 ], "matrix": "L11.mlp.W_out" }, { "axis0": [ 101, 100 ], "axis1": [ 331, 20 ], "matrix": "L11.mlp.W_out" } ], "l:83": [ { "axis0": [ 239, 1 ], "axis1": [ 128, 64 ], "matrix": "L11.attn.W_Q" }, { "axis0": [ 258, 1 ], "axis1": [ 128, 64 ], "matrix": "L11.attn.W_Q" }, { "axis0": [ 21, 1 ], "axis1": [ 128, 64 ], "matrix": "L11.attn.W_K" }, { "axis0": [ 259, 1 ], "axis1": [ 128, 64 ], "matrix": "L11.attn.W_K" }, { "axis0": [ 24, 2 ], "axis1": [ 128, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 66, 1 ], "axis1": [ 128, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 137, 2 ], "axis1": [ 128, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 173, 12 ], "axis1": [ 128, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 174, 16 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 210, 1 ], "matrix": "L11.attn.W_O" } ], "l:85": [ { "axis0": [ 0, 1 ], "axis1": [ 258, 1 ], "matrix": "L10.mlp.W_out" } ], "l:86": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 1 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 239, 1 ], "matrix": "L8.mlp.W_out" } ], "l:88": [ { "axis0": [ 0, 1 ], "axis1": [ 259, 1 ], "matrix": "L10.mlp.W_out" } ], "l:89": [ { "axis0": [ 22, 1 ], "axis1": [ 64, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 25, 1 ], "axis1": [ 64, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 137, 2 ], "axis1": [ 64, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 42, 4 ], "axis1": [ 64, 64 ], "matrix": "L8.attn.W_K" }, { "axis0": [ 24, 1 ], "axis1": [ 64, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 173, 16 ], "axis1": [ 64, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 24, 2 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 66, 1 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 137, 2 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 173, 12 ], "matrix": "L8.attn.W_O" } ], "l:9": [ { "axis0": [ 0, 1 ], "axis1": [ 41, 1 ], "matrix": "L0.mlp.W_out" } ], "l:90": [ { "axis0": [ 0, 1 ], "axis1": [ 51, 9 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 51, 9 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 22, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 25, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 137, 2 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 32, 1 ], "axis1": [ 51, 9 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 51, 9 ], "axis1": [ 22, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 51, 9 ], "axis1": [ 25, 1 ], "matrix": "L4.mlp.W_out" }, { "axis0": [ 51, 9 ], "axis1": [ 137, 2 ], "matrix": "L4.mlp.W_out" } ], "l:91": [ { "axis0": [ 57, 2 ], "axis1": [ 128, 64 ], "matrix": "L4.attn.W_Q" }, { "axis0": [ 27, 1 ], "axis1": [ 128, 64 ], "matrix": "L4.attn.W_K" }, { "axis0": [ 59, 1 ], "axis1": [ 128, 64 ], "matrix": "L4.attn.W_K" }, { "axis0": [ 22, 1 ], "axis1": [ 128, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 32, 1 ], "matrix": "L4.attn.W_O" } ], "l:93": [ { "axis0": [ 0, 1 ], "axis1": [ 57, 1 ], "matrix": "L3.mlp.W_out" } ], "l:94": [ { "axis0": [ 0, 1 ], "axis1": [ 58, 1 ], "matrix": "L3.mlp.W_out" } ], "l:96": [ { "axis0": [ 0, 1 ], "axis1": [ 59, 1 ], "matrix": "L3.mlp.W_out" } ], "l:97": [ { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L3.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L3.attn.W_K" }, { "axis0": [ 33, 1 ], "axis1": [ 0, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 22, 1 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 22, 1 ], "matrix": "L3.mlp.W_out" } ], "l:98": [ { "axis0": [ 0, 1 ], "axis1": [ 3, 32 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 0, 1 ], "axis1": [ 3, 32 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 0, 1 ], "axis1": [ 33, 1 ], "matrix": "L2.mlp.W_out" }, { "axis0": [ 22, 1 ], "axis1": [ 3, 32 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 3, 32 ], "axis1": [ 33, 1 ], "matrix": "L2.mlp.W_out" } ], "l:99": [ { "axis0": [ 33, 1 ], "axis1": [ 0, 64 ], "matrix": "L2.attn.W_Q" }, { "axis0": [ 25, 1 ], "axis1": [ 0, 64 ], "matrix": "L2.attn.W_K" }, { "axis0": [ 46, 1 ], "axis1": [ 0, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 22, 1 ], "matrix": "L2.attn.W_O" } ], "physical:cancel": [ { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L1.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L1.attn.W_K" }, { "axis0": [ 24, 6 ], "axis1": [ 192, 64 ], "matrix": "L1.attn.W_V" }, { "axis0": [ 39, 3 ], "axis1": [ 192, 64 ], "matrix": "L1.attn.W_V" }, { "axis0": [ 192, 64 ], "axis1": [ 24, 6 ], "matrix": "L1.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 39, 3 ], "matrix": "L1.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L2.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L2.attn.W_K" }, { "axis0": [ 24, 2 ], "axis1": [ 128, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 33, 1 ], "axis1": [ 128, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 46, 3 ], "axis1": [ 128, 64 ], "matrix": "L2.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 24, 2 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 33, 1 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 46, 3 ], "matrix": "L2.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L3.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L3.attn.W_K" }, { "axis0": [ 22, 1 ], "axis1": [ 128, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 24, 1 ], "axis1": [ 128, 64 ], "matrix": "L3.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 22, 1 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 24, 1 ], "matrix": "L3.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 448, 64 ], "matrix": "L4.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 448, 64 ], "matrix": "L4.attn.W_K" }, { "axis0": [ 18, 3 ], "axis1": [ 448, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 22, 1 ], "axis1": [ 448, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 30, 3 ], "axis1": [ 448, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 50, 3 ], "axis1": [ 448, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 57, 12 ], "axis1": [ 448, 64 ], "matrix": "L4.attn.W_V" }, { "axis0": [ 448, 64 ], "axis1": [ 18, 3 ], "matrix": "L4.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 22, 1 ], "matrix": "L4.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 30, 3 ], "matrix": "L4.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 50, 3 ], "matrix": "L4.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 57, 12 ], "matrix": "L4.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L5.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L5.attn.W_K" }, { "axis0": [ 28, 5 ], "axis1": [ 192, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 34, 1 ], "axis1": [ 192, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 36, 3 ], "axis1": [ 192, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 46, 1 ], "axis1": [ 192, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 50, 1 ], "axis1": [ 192, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 190, 17 ], "axis1": [ 192, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 224, 17 ], "axis1": [ 192, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 288, 1 ], "axis1": [ 192, 64 ], "matrix": "L5.attn.W_V" }, { "axis0": [ 192, 64 ], "axis1": [ 28, 5 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 34, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 36, 3 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 46, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 50, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 190, 17 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 224, 17 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 288, 1 ], "matrix": "L5.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L6.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L6.attn.W_K" }, { "axis0": [ 24, 1 ], "axis1": [ 64, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 102, 1 ], "axis1": [ 64, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 278, 1 ], "axis1": [ 64, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 280, 2 ], "axis1": [ 64, 64 ], "matrix": "L6.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 24, 1 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 102, 1 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 278, 1 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 280, 2 ], "matrix": "L6.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L7.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L7.attn.W_K" }, { "axis0": [ 1, 17 ], "axis1": [ 192, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 51, 1 ], "axis1": [ 192, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 207, 3 ], "axis1": [ 192, 64 ], "matrix": "L7.attn.W_V" }, { "axis0": [ 192, 64 ], "axis1": [ 1, 17 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 51, 1 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 207, 3 ], "matrix": "L7.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 256, 64 ], "matrix": "L8.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 256, 64 ], "matrix": "L8.attn.W_K" }, { "axis0": [ 18, 3 ], "axis1": [ 256, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 22, 4 ], "axis1": [ 256, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 42, 4 ], "axis1": [ 256, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 50, 5 ], "axis1": [ 256, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 64, 5 ], "axis1": [ 256, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 102, 1 ], "axis1": [ 256, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 137, 2 ], "axis1": [ 256, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 173, 16 ], "axis1": [ 256, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 210, 1 ], "axis1": [ 256, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 326, 1 ], "axis1": [ 256, 64 ], "matrix": "L8.attn.W_V" }, { "axis0": [ 256, 64 ], "axis1": [ 18, 3 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 22, 4 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 42, 4 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 50, 5 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 64, 5 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 102, 1 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 137, 2 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 173, 16 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 210, 1 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 326, 1 ], "matrix": "L8.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L9.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L9.attn.W_K" }, { "axis0": [ 45, 1 ], "axis1": [ 64, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 222, 2 ], "axis1": [ 64, 64 ], "matrix": "L9.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 45, 1 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 222, 2 ], "matrix": "L9.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L10.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L10.attn.W_K" }, { "axis0": [ 55, 2 ], "axis1": [ 192, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 60, 4 ], "axis1": [ 192, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 103, 18 ], "axis1": [ 192, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 207, 3 ], "axis1": [ 192, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 211, 2 ], "axis1": [ 192, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 236, 3 ], "axis1": [ 192, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 327, 4 ], "axis1": [ 192, 64 ], "matrix": "L10.attn.W_V" }, { "axis0": [ 192, 64 ], "axis1": [ 55, 2 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 60, 4 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 103, 18 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 207, 3 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 211, 2 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 236, 3 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 327, 4 ], "matrix": "L10.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 384, 64 ], "matrix": "L11.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 384, 64 ], "matrix": "L11.attn.W_K" }, { "axis0": [ 18, 8 ], "axis1": [ 384, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 42, 4 ], "axis1": [ 384, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 50, 7 ], "axis1": [ 384, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 59, 10 ], "axis1": [ 384, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 102, 9 ], "axis1": [ 384, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 137, 2 ], "axis1": [ 384, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 173, 17 ], "axis1": [ 384, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 210, 1 ], "axis1": [ 384, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 213, 6 ], "axis1": [ 384, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 384, 64 ], "axis1": [ 18, 8 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 42, 4 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 50, 7 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 59, 10 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 102, 9 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 137, 2 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 173, 17 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 210, 1 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 384, 64 ], "axis1": [ 213, 6 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 448, 64 ], "matrix": "L11.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 448, 64 ], "matrix": "L11.attn.W_K" }, { "axis0": [ 219, 3 ], "axis1": [ 448, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 223, 1 ], "axis1": [ 448, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 236, 20 ], "axis1": [ 448, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 257, 6 ], "axis1": [ 448, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 266, 3 ], "axis1": [ 448, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 326, 1 ], "axis1": [ 448, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 328, 2 ], "axis1": [ 448, 64 ], "matrix": "L11.attn.W_V" }, { "axis0": [ 448, 64 ], "axis1": [ 219, 3 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 223, 1 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 236, 20 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 257, 6 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 266, 3 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 326, 1 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 448, 64 ], "axis1": [ 328, 2 ], "matrix": "L11.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 576, 64 ], "matrix": "L12.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 576, 64 ], "matrix": "L12.attn.W_K" }, { "axis0": [ 351, 34 ], "axis1": [ 576, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 419, 30 ], "axis1": [ 576, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 576, 64 ], "axis1": [ 351, 34 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 576, 64 ], "axis1": [ 419, 30 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 640, 64 ], "matrix": "L12.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 640, 64 ], "matrix": "L12.attn.W_K" }, { "axis0": [ 449, 38 ], "axis1": [ 640, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 521, 26 ], "axis1": [ 640, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 640, 64 ], "axis1": [ 449, 38 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 640, 64 ], "axis1": [ 521, 26 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 704, 64 ], "matrix": "L12.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 704, 64 ], "matrix": "L12.attn.W_K" }, { "axis0": [ 547, 8 ], "axis1": [ 704, 64 ], "matrix": "L12.attn.W_V" }, { "axis0": [ 704, 64 ], "axis1": [ 547, 8 ], "matrix": "L12.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L13.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L13.attn.W_K" }, { "axis0": [ 211, 1 ], "axis1": [ 64, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 222, 1 ], "axis1": [ 64, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 263, 1 ], "axis1": [ 64, 64 ], "matrix": "L13.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 211, 1 ], "matrix": "L13.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 222, 1 ], "matrix": "L13.attn.W_O" }, { "axis0": [ 64, 64 ], "axis1": [ 263, 1 ], "matrix": "L13.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L14.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L14.attn.W_K" }, { "axis0": [ 360, 2 ], "axis1": [ 64, 64 ], "matrix": "L14.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 360, 2 ], "matrix": "L14.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 256, 64 ], "matrix": "L15.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 256, 64 ], "matrix": "L15.attn.W_K" }, { "axis0": [ 356, 1 ], "axis1": [ 256, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 468, 1 ], "axis1": [ 256, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 743, 62 ], "axis1": [ 256, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 256, 64 ], "axis1": [ 356, 1 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 468, 1 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 743, 62 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 320, 64 ], "matrix": "L15.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 320, 64 ], "matrix": "L15.attn.W_K" }, { "axis0": [ 805, 38 ], "axis1": [ 320, 64 ], "matrix": "L15.attn.W_V" }, { "axis0": [ 320, 64 ], "axis1": [ 805, 38 ], "matrix": "L15.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L16.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L16.attn.W_K" }, { "axis0": [ 68, 1 ], "axis1": [ 192, 64 ], "matrix": "L16.attn.W_V" }, { "axis0": [ 192, 64 ], "axis1": [ 68, 1 ], "matrix": "L16.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L17.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L17.attn.W_K" }, { "axis0": [ 21, 3 ], "axis1": [ 192, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 33, 1 ], "axis1": [ 192, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 43, 1 ], "axis1": [ 192, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 68, 1 ], "axis1": [ 192, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 103, 8 ], "axis1": [ 192, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 120, 1 ], "axis1": [ 192, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 218, 2 ], "axis1": [ 192, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 327, 1 ], "axis1": [ 192, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 347, 1 ], "axis1": [ 192, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 356, 1 ], "axis1": [ 192, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 395, 1 ], "axis1": [ 192, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 515, 4 ], "axis1": [ 192, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 520, 2 ], "axis1": [ 192, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 525, 9 ], "axis1": [ 192, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 535, 3 ], "axis1": [ 192, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 540, 2 ], "axis1": [ 192, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 726, 17 ], "axis1": [ 192, 64 ], "matrix": "L17.attn.W_V" }, { "axis0": [ 192, 64 ], "axis1": [ 21, 3 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 33, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 43, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 68, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 103, 8 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 120, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 218, 2 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 327, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 347, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 356, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 395, 1 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 515, 4 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 520, 2 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 525, 9 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 535, 3 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 540, 2 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 726, 17 ], "matrix": "L17.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L18.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L18.attn.W_K" }, { "axis0": [ 43, 1 ], "axis1": [ 192, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 57, 1 ], "axis1": [ 192, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 60, 3 ], "axis1": [ 192, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 65, 1 ], "axis1": [ 192, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 273, 12 ], "axis1": [ 192, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 288, 2 ], "axis1": [ 192, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 299, 5 ], "axis1": [ 192, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 305, 2 ], "axis1": [ 192, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 308, 4 ], "axis1": [ 192, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 313, 1 ], "axis1": [ 192, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 315, 1 ], "axis1": [ 192, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 326, 1 ], "axis1": [ 192, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 337, 3 ], "axis1": [ 192, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 393, 1 ], "axis1": [ 192, 64 ], "matrix": "L18.attn.W_V" }, { "axis0": [ 192, 64 ], "axis1": [ 43, 1 ], "matrix": "L18.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 57, 1 ], "matrix": "L18.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 60, 3 ], "matrix": "L18.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 65, 1 ], "matrix": "L18.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 273, 12 ], "matrix": "L18.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 288, 2 ], "matrix": "L18.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 299, 5 ], "matrix": "L18.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 305, 2 ], "matrix": "L18.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 308, 4 ], "matrix": "L18.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 313, 1 ], "matrix": "L18.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 315, 1 ], "matrix": "L18.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 326, 1 ], "matrix": "L18.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 337, 3 ], "matrix": "L18.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 393, 1 ], "matrix": "L18.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L19.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L19.attn.W_K" }, { "axis0": [ 43, 1 ], "axis1": [ 128, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 51, 1 ], "axis1": [ 128, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 55, 1 ], "axis1": [ 128, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 65, 1 ], "axis1": [ 128, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 67, 1 ], "axis1": [ 128, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 258, 6 ], "axis1": [ 128, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 266, 6 ], "axis1": [ 128, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 273, 1 ], "axis1": [ 128, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 326, 1 ], "axis1": [ 128, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 336, 1 ], "axis1": [ 128, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 391, 1 ], "axis1": [ 128, 64 ], "matrix": "L19.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 43, 1 ], "matrix": "L19.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 51, 1 ], "matrix": "L19.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 55, 1 ], "matrix": "L19.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 65, 1 ], "matrix": "L19.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 67, 1 ], "matrix": "L19.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 258, 6 ], "matrix": "L19.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 266, 6 ], "matrix": "L19.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 273, 1 ], "matrix": "L19.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 326, 1 ], "matrix": "L19.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 336, 1 ], "matrix": "L19.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 391, 1 ], "matrix": "L19.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L20.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 128, 64 ], "matrix": "L20.attn.W_K" }, { "axis0": [ 29, 4 ], "axis1": [ 128, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 34, 1 ], "axis1": [ 128, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 36, 4 ], "axis1": [ 128, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 49, 2 ], "axis1": [ 128, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 56, 1 ], "axis1": [ 128, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 58, 2 ], "axis1": [ 128, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 63, 1 ], "axis1": [ 128, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 65, 1 ], "axis1": [ 128, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 156, 17 ], "axis1": [ 128, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 190, 8 ], "axis1": [ 128, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 240, 7 ], "axis1": [ 128, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 248, 3 ], "axis1": [ 128, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 252, 4 ], "axis1": [ 128, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 290, 9 ], "axis1": [ 128, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 128, 64 ], "axis1": [ 29, 4 ], "matrix": "L20.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 34, 1 ], "matrix": "L20.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 36, 4 ], "matrix": "L20.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 49, 2 ], "matrix": "L20.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 56, 1 ], "matrix": "L20.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 58, 2 ], "matrix": "L20.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 63, 1 ], "matrix": "L20.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 65, 1 ], "matrix": "L20.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 156, 17 ], "matrix": "L20.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 190, 8 ], "matrix": "L20.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 240, 7 ], "matrix": "L20.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 248, 3 ], "matrix": "L20.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 252, 4 ], "matrix": "L20.attn.W_O" }, { "axis0": [ 128, 64 ], "axis1": [ 290, 9 ], "matrix": "L20.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L20.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L20.attn.W_K" }, { "axis0": [ 316, 1 ], "axis1": [ 192, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 318, 1 ], "axis1": [ 192, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 320, 1 ], "axis1": [ 192, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 326, 1 ], "axis1": [ 192, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 328, 1 ], "axis1": [ 192, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 335, 1 ], "axis1": [ 192, 64 ], "matrix": "L20.attn.W_V" }, { "axis0": [ 192, 64 ], "axis1": [ 316, 1 ], "matrix": "L20.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 318, 1 ], "matrix": "L20.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 320, 1 ], "matrix": "L20.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 326, 1 ], "matrix": "L20.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 328, 1 ], "matrix": "L20.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 335, 1 ], "matrix": "L20.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L23.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L23.attn.W_K" }, { "axis0": [ 121, 1 ], "axis1": [ 0, 64 ], "matrix": "L23.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 121, 1 ], "matrix": "L23.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L24.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 64, 64 ], "matrix": "L24.attn.W_K" }, { "axis0": [ 121, 4 ], "axis1": [ 64, 64 ], "matrix": "L24.attn.W_V" }, { "axis0": [ 64, 64 ], "axis1": [ 121, 4 ], "matrix": "L24.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L25.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L25.attn.W_K" }, { "axis0": [ 364, 1 ], "axis1": [ 0, 64 ], "matrix": "L25.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 364, 1 ], "matrix": "L25.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L26.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 0, 64 ], "matrix": "L26.attn.W_K" }, { "axis0": [ 121, 1 ], "axis1": [ 0, 64 ], "matrix": "L26.attn.W_V" }, { "axis0": [ 0, 64 ], "axis1": [ 121, 1 ], "matrix": "L26.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L28.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 192, 64 ], "matrix": "L28.attn.W_K" }, { "axis0": [ 18, 1 ], "axis1": [ 192, 64 ], "matrix": "L28.attn.W_V" }, { "axis0": [ 27, 2 ], "axis1": [ 192, 64 ], "matrix": "L28.attn.W_V" }, { "axis0": [ 32, 7 ], "axis1": [ 192, 64 ], "matrix": "L28.attn.W_V" }, { "axis0": [ 40, 1 ], "axis1": [ 192, 64 ], "matrix": "L28.attn.W_V" }, { "axis0": [ 42, 2 ], "axis1": [ 192, 64 ], "matrix": "L28.attn.W_V" }, { "axis0": [ 65, 1 ], "axis1": [ 192, 64 ], "matrix": "L28.attn.W_V" }, { "axis0": [ 69, 2 ], "axis1": [ 192, 64 ], "matrix": "L28.attn.W_V" }, { "axis0": [ 72, 2 ], "axis1": [ 192, 64 ], "matrix": "L28.attn.W_V" }, { "axis0": [ 79, 1 ], "axis1": [ 192, 64 ], "matrix": "L28.attn.W_V" }, { "axis0": [ 81, 21 ], "axis1": [ 192, 64 ], "matrix": "L28.attn.W_V" }, { "axis0": [ 122, 15 ], "axis1": [ 192, 64 ], "matrix": "L28.attn.W_V" }, { "axis0": [ 139, 6 ], "axis1": [ 192, 64 ], "matrix": "L28.attn.W_V" }, { "axis0": [ 162, 3 ], "axis1": [ 192, 64 ], "matrix": "L28.attn.W_V" }, { "axis0": [ 192, 64 ], "axis1": [ 18, 1 ], "matrix": "L28.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 27, 2 ], "matrix": "L28.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 32, 7 ], "matrix": "L28.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 40, 1 ], "matrix": "L28.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 42, 2 ], "matrix": "L28.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 65, 1 ], "matrix": "L28.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 69, 2 ], "matrix": "L28.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 72, 2 ], "matrix": "L28.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 79, 1 ], "matrix": "L28.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 81, 21 ], "matrix": "L28.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 122, 15 ], "matrix": "L28.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 139, 6 ], "matrix": "L28.attn.W_O" }, { "axis0": [ 192, 64 ], "axis1": [ 162, 3 ], "matrix": "L28.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 256, 64 ], "matrix": "L28.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 256, 64 ], "matrix": "L28.attn.W_K" }, { "axis0": [ 165, 24 ], "axis1": [ 256, 64 ], "matrix": "L28.attn.W_V" }, { "axis0": [ 190, 4 ], "axis1": [ 256, 64 ], "matrix": "L28.attn.W_V" }, { "axis0": [ 197, 20 ], "axis1": [ 256, 64 ], "matrix": "L28.attn.W_V" }, { "axis0": [ 232, 1 ], "axis1": [ 256, 64 ], "matrix": "L28.attn.W_V" }, { "axis0": [ 326, 2 ], "axis1": [ 256, 64 ], "matrix": "L28.attn.W_V" }, { "axis0": [ 329, 1 ], "axis1": [ 256, 64 ], "matrix": "L28.attn.W_V" }, { "axis0": [ 333, 2 ], "axis1": [ 256, 64 ], "matrix": "L28.attn.W_V" }, { "axis0": [ 392, 1 ], "axis1": [ 256, 64 ], "matrix": "L28.attn.W_V" }, { "axis0": [ 404, 1 ], "axis1": [ 256, 64 ], "matrix": "L28.attn.W_V" }, { "axis0": [ 1931, 8 ], "axis1": [ 256, 64 ], "matrix": "L28.attn.W_V" }, { "axis0": [ 256, 64 ], "axis1": [ 165, 24 ], "matrix": "L28.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 190, 4 ], "matrix": "L28.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 197, 20 ], "matrix": "L28.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 232, 1 ], "matrix": "L28.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 326, 2 ], "matrix": "L28.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 329, 1 ], "matrix": "L28.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 333, 2 ], "matrix": "L28.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 392, 1 ], "matrix": "L28.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 404, 1 ], "matrix": "L28.attn.W_O" }, { "axis0": [ 256, 64 ], "axis1": [ 1931, 8 ], "matrix": "L28.attn.W_O" }, { "axis0": [ 0, 1 ], "axis1": [ 320, 64 ], "matrix": "L28.attn.W_Q" }, { "axis0": [ 0, 1 ], "axis1": [ 320, 64 ], "matrix": "L28.attn.W_K" }, { "axis0": [ 1939, 9 ], "axis1": [ 320, 64 ], "matrix": "L28.attn.W_V" }, { "axis0": [ 320, 64 ], "axis1": [ 1939, 9 ], "matrix": "L28.attn.W_O" } ], "physical:cancel_bypass": [ { "axis0": [ 0, 1 ], "axis1": [ 19, 8 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 35, 4 ], "axis1": [ 19, 4 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 35, 4 ], "axis1": [ 23, 4 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 19, 4 ], "axis1": [ 35, 4 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 23, 4 ], "axis1": [ 35, 4 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 27, 2 ], "matrix": "L1.mlp.W_up" }, { "axis0": [ 22, 1 ], "axis1": [ 27, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 22, 1 ], "axis1": [ 28, 1 ], "matrix": "L1.mlp.W_in" }, { "axis0": [ 27, 1 ], "axis1": [ 22, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 28, 1 ], "axis1": [ 22, 1 ], "diagonal": true, "matrix": "L1.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 67, 2 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 27, 1 ], "axis1": [ 67, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 27, 1 ], "axis1": [ 68, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 67, 1 ], "axis1": [ 27, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 68, 1 ], "axis1": [ 27, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 69, 2 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 28, 1 ], "axis1": [ 69, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 28, 1 ], "axis1": [ 70, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 69, 1 ], "axis1": [ 28, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 70, 1 ], "axis1": [ 28, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 71, 2 ], "matrix": "L2.mlp.W_up" }, { "axis0": [ 29, 1 ], "axis1": [ 71, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 29, 1 ], "axis1": [ 72, 1 ], "matrix": "L2.mlp.W_in" }, { "axis0": [ 71, 1 ], "axis1": [ 29, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 72, 1 ], "axis1": [ 29, 1 ], "diagonal": true, "matrix": "L2.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 17, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 25, 1 ], "axis1": [ 17, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 25, 1 ], "axis1": [ 18, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 17, 1 ], "axis1": [ 25, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 18, 1 ], "axis1": [ 25, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 19, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 40, 1 ], "axis1": [ 19, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 40, 1 ], "axis1": [ 20, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 19, 1 ], "axis1": [ 40, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 20, 1 ], "axis1": [ 40, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 21, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 33, 1 ], "axis1": [ 21, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 33, 1 ], "axis1": [ 22, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 21, 1 ], "axis1": [ 33, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 22, 1 ], "axis1": [ 33, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 23, 2 ], "matrix": "L3.mlp.W_up" }, { "axis0": [ 35, 1 ], "axis1": [ 23, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 35, 1 ], "axis1": [ 24, 1 ], "matrix": "L3.mlp.W_in" }, { "axis0": [ 23, 1 ], "axis1": [ 35, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 24, 1 ], "axis1": [ 35, 1 ], "diagonal": true, "matrix": "L3.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 224, 34 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 69, 17 ], "axis1": [ 224, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 69, 17 ], "axis1": [ 241, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 224, 17 ], "axis1": [ 69, 17 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 241, 17 ], "axis1": [ 69, 17 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 258, 34 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 86, 17 ], "axis1": [ 258, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 86, 17 ], "axis1": [ 275, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 258, 17 ], "axis1": [ 86, 17 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 275, 17 ], "axis1": [ 86, 17 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 292, 34 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 103, 17 ], "axis1": [ 292, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 103, 17 ], "axis1": [ 309, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 292, 17 ], "axis1": [ 103, 17 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 309, 17 ], "axis1": [ 103, 17 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 326, 34 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 120, 17 ], "axis1": [ 326, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 120, 17 ], "axis1": [ 343, 17 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 326, 17 ], "axis1": [ 120, 17 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 343, 17 ], "axis1": [ 120, 17 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 360, 8 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 53, 4 ], "axis1": [ 360, 4 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 53, 4 ], "axis1": [ 364, 4 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 360, 4 ], "axis1": [ 53, 4 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 364, 4 ], "axis1": [ 53, 4 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 368, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 27, 1 ], "axis1": [ 368, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 27, 1 ], "axis1": [ 369, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 368, 1 ], "axis1": [ 27, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 369, 1 ], "axis1": [ 27, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 370, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 39, 1 ], "axis1": [ 370, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 39, 1 ], "axis1": [ 371, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 370, 1 ], "axis1": [ 39, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 371, 1 ], "axis1": [ 39, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 372, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 49, 1 ], "axis1": [ 372, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 49, 1 ], "axis1": [ 373, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 372, 1 ], "axis1": [ 49, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 373, 1 ], "axis1": [ 49, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 374, 2 ], "matrix": "L4.mlp.W_up" }, { "axis0": [ 24, 1 ], "axis1": [ 374, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 24, 1 ], "axis1": [ 375, 1 ], "matrix": "L4.mlp.W_in" }, { "axis0": [ 374, 1 ], "axis1": [ 24, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 375, 1 ], "axis1": [ 24, 1 ], "diagonal": true, "matrix": "L4.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 136, 34 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 289, 17 ], "axis1": [ 136, 17 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 289, 17 ], "axis1": [ 153, 17 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 136, 17 ], "axis1": [ 289, 17 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 153, 17 ], "axis1": [ 289, 17 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 170, 34 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 306, 17 ], "axis1": [ 170, 17 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 306, 17 ], "axis1": [ 187, 17 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 170, 17 ], "axis1": [ 306, 17 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 187, 17 ], "axis1": [ 306, 17 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 204, 34 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 173, 17 ], "axis1": [ 204, 17 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 173, 17 ], "axis1": [ 221, 17 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 204, 17 ], "axis1": [ 173, 17 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 221, 17 ], "axis1": [ 173, 17 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 238, 34 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 207, 17 ], "axis1": [ 238, 17 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 207, 17 ], "axis1": [ 255, 17 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 238, 17 ], "axis1": [ 207, 17 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 255, 17 ], "axis1": [ 207, 17 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 272, 34 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 241, 17 ], "axis1": [ 272, 17 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 241, 17 ], "axis1": [ 289, 17 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 272, 17 ], "axis1": [ 241, 17 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 289, 17 ], "axis1": [ 241, 17 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 306, 34 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 258, 17 ], "axis1": [ 306, 17 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 258, 17 ], "axis1": [ 323, 17 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 306, 17 ], "axis1": [ 258, 17 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 323, 17 ], "axis1": [ 258, 17 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 340, 6 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 52, 1 ], "axis1": [ 340, 3 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 57, 2 ], "axis1": [ 340, 3 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 52, 1 ], "axis1": [ 343, 3 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 57, 2 ], "axis1": [ 343, 3 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 340, 1 ], "axis1": [ 52, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 341, 2 ], "axis1": [ 57, 2 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 343, 1 ], "axis1": [ 52, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 344, 2 ], "axis1": [ 57, 2 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 346, 2 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 23, 1 ], "axis1": [ 346, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 23, 1 ], "axis1": [ 347, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 346, 1 ], "axis1": [ 23, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 347, 1 ], "axis1": [ 23, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 348, 2 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 51, 1 ], "axis1": [ 348, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 51, 1 ], "axis1": [ 349, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 348, 1 ], "axis1": [ 51, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 349, 1 ], "axis1": [ 51, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 350, 2 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 275, 1 ], "axis1": [ 350, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 275, 1 ], "axis1": [ 351, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 350, 1 ], "axis1": [ 275, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 351, 1 ], "axis1": [ 275, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 352, 2 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 276, 1 ], "axis1": [ 352, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 276, 1 ], "axis1": [ 353, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 352, 1 ], "axis1": [ 276, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 353, 1 ], "axis1": [ 276, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 354, 2 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 277, 1 ], "axis1": [ 354, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 277, 1 ], "axis1": [ 355, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 354, 1 ], "axis1": [ 277, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 355, 1 ], "axis1": [ 277, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 356, 2 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 279, 1 ], "axis1": [ 356, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 279, 1 ], "axis1": [ 357, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 356, 1 ], "axis1": [ 279, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 357, 1 ], "axis1": [ 279, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 358, 2 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 282, 1 ], "axis1": [ 358, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 282, 1 ], "axis1": [ 359, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 358, 1 ], "axis1": [ 282, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 359, 1 ], "axis1": [ 282, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 360, 2 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 283, 1 ], "axis1": [ 360, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 283, 1 ], "axis1": [ 361, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 360, 1 ], "axis1": [ 283, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 361, 1 ], "axis1": [ 283, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 362, 2 ], "matrix": "L5.mlp.W_up" }, { "axis0": [ 284, 1 ], "axis1": [ 362, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 284, 1 ], "axis1": [ 363, 1 ], "matrix": "L5.mlp.W_in" }, { "axis0": [ 362, 1 ], "axis1": [ 284, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 363, 1 ], "axis1": [ 284, 1 ], "diagonal": true, "matrix": "L5.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 40, 34 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 49, 2 ], "axis1": [ 40, 17 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 53, 4 ], "axis1": [ 40, 17 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 239, 2 ], "axis1": [ 40, 17 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 288, 1 ], "axis1": [ 40, 17 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 323, 8 ], "axis1": [ 40, 17 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 49, 2 ], "axis1": [ 57, 17 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 53, 4 ], "axis1": [ 57, 17 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 239, 2 ], "axis1": [ 57, 17 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 288, 1 ], "axis1": [ 57, 17 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 323, 8 ], "axis1": [ 57, 17 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 40, 2 ], "axis1": [ 49, 2 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 42, 4 ], "axis1": [ 53, 4 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 46, 2 ], "axis1": [ 239, 2 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 48, 1 ], "axis1": [ 288, 1 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 49, 8 ], "axis1": [ 323, 8 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 57, 2 ], "axis1": [ 49, 2 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 59, 4 ], "axis1": [ 53, 4 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 63, 2 ], "axis1": [ 239, 2 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 65, 1 ], "axis1": [ 288, 1 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 66, 8 ], "axis1": [ 323, 8 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 74, 6 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 236, 3 ], "axis1": [ 74, 3 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 236, 3 ], "axis1": [ 77, 3 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 74, 3 ], "axis1": [ 236, 3 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 77, 3 ], "axis1": [ 236, 3 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 80, 2 ], "matrix": "L6.mlp.W_up" }, { "axis0": [ 103, 1 ], "axis1": [ 80, 1 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 103, 1 ], "axis1": [ 81, 1 ], "matrix": "L6.mlp.W_in" }, { "axis0": [ 80, 1 ], "axis1": [ 103, 1 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 81, 1 ], "axis1": [ 103, 1 ], "diagonal": true, "matrix": "L6.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 52, 2 ], "matrix": "L7.mlp.W_up" }, { "axis0": [ 189, 1 ], "axis1": [ 52, 1 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 189, 1 ], "axis1": [ 53, 1 ], "matrix": "L7.mlp.W_in" }, { "axis0": [ 52, 1 ], "axis1": [ 189, 1 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 53, 1 ], "axis1": [ 189, 1 ], "diagonal": true, "matrix": "L7.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 4, 2 ], "matrix": "L8.mlp.W_up" }, { "axis0": [ 59, 1 ], "axis1": [ 4, 1 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 59, 1 ], "axis1": [ 5, 1 ], "matrix": "L8.mlp.W_in" }, { "axis0": [ 4, 1 ], "axis1": [ 59, 1 ], "diagonal": true, "matrix": "L8.mlp.W_out" }, { "axis0": [ 5, 1 ], "axis1": [ 59, 1 ], "diagonal": true, "matrix": "L8.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1, 2 ], "matrix": "L10.mlp.W_up" }, { "axis0": [ 49, 1 ], "axis1": [ 1, 1 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 49, 1 ], "axis1": [ 2, 1 ], "matrix": "L10.mlp.W_in" }, { "axis0": [ 1, 1 ], "axis1": [ 49, 1 ], "diagonal": true, "matrix": "L10.mlp.W_out" }, { "axis0": [ 2, 1 ], "axis1": [ 49, 1 ], "diagonal": true, "matrix": "L10.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1501, 34 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 111, 10 ], "axis1": [ 1501, 17 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 207, 3 ], "axis1": [ 1501, 17 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 211, 2 ], "axis1": [ 1501, 17 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 222, 1 ], "axis1": [ 1501, 17 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 327, 1 ], "axis1": [ 1501, 17 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 111, 10 ], "axis1": [ 1518, 17 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 207, 3 ], "axis1": [ 1518, 17 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 211, 2 ], "axis1": [ 1518, 17 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 222, 1 ], "axis1": [ 1518, 17 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 327, 1 ], "axis1": [ 1518, 17 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 1501, 10 ], "axis1": [ 111, 10 ], "diagonal": true, "matrix": "L11.mlp.W_out" }, { "axis0": [ 1511, 3 ], "axis1": [ 207, 3 ], "diagonal": true, "matrix": "L11.mlp.W_out" }, { "axis0": [ 1514, 2 ], "axis1": [ 211, 2 ], "diagonal": true, "matrix": "L11.mlp.W_out" }, { "axis0": [ 1516, 1 ], "axis1": [ 222, 1 ], "diagonal": true, "matrix": "L11.mlp.W_out" }, { "axis0": [ 1517, 1 ], "axis1": [ 327, 1 ], "diagonal": true, "matrix": "L11.mlp.W_out" }, { "axis0": [ 1518, 10 ], "axis1": [ 111, 10 ], "diagonal": true, "matrix": "L11.mlp.W_out" }, { "axis0": [ 1528, 3 ], "axis1": [ 207, 3 ], "diagonal": true, "matrix": "L11.mlp.W_out" }, { "axis0": [ 1531, 2 ], "axis1": [ 211, 2 ], "diagonal": true, "matrix": "L11.mlp.W_out" }, { "axis0": [ 1533, 1 ], "axis1": [ 222, 1 ], "diagonal": true, "matrix": "L11.mlp.W_out" }, { "axis0": [ 1534, 1 ], "axis1": [ 327, 1 ], "diagonal": true, "matrix": "L11.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1535, 2 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 263, 1 ], "axis1": [ 1535, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 263, 1 ], "axis1": [ 1536, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 1535, 1 ], "axis1": [ 263, 1 ], "diagonal": true, "matrix": "L11.mlp.W_out" }, { "axis0": [ 1536, 1 ], "axis1": [ 263, 1 ], "diagonal": true, "matrix": "L11.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1537, 2 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 264, 1 ], "axis1": [ 1537, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 264, 1 ], "axis1": [ 1538, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 1537, 1 ], "axis1": [ 264, 1 ], "diagonal": true, "matrix": "L11.mlp.W_out" }, { "axis0": [ 1538, 1 ], "axis1": [ 264, 1 ], "diagonal": true, "matrix": "L11.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1539, 2 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 265, 1 ], "axis1": [ 1539, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 265, 1 ], "axis1": [ 1540, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 1539, 1 ], "axis1": [ 265, 1 ], "diagonal": true, "matrix": "L11.mlp.W_out" }, { "axis0": [ 1540, 1 ], "axis1": [ 265, 1 ], "diagonal": true, "matrix": "L11.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1541, 2 ], "matrix": "L11.mlp.W_up" }, { "axis0": [ 256, 1 ], "axis1": [ 1541, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 256, 1 ], "axis1": [ 1542, 1 ], "matrix": "L11.mlp.W_in" }, { "axis0": [ 1541, 1 ], "axis1": [ 256, 1 ], "diagonal": true, "matrix": "L11.mlp.W_out" }, { "axis0": [ 1542, 1 ], "axis1": [ 256, 1 ], "diagonal": true, "matrix": "L11.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 41, 68 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 308, 11 ], "axis1": [ 41, 34 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 320, 3 ], "axis1": [ 41, 34 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 331, 20 ], "axis1": [ 41, 34 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 308, 11 ], "axis1": [ 75, 34 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 320, 3 ], "axis1": [ 75, 34 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 331, 20 ], "axis1": [ 75, 34 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 41, 11 ], "axis1": [ 308, 11 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 52, 3 ], "axis1": [ 320, 3 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 55, 20 ], "axis1": [ 331, 20 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 75, 11 ], "axis1": [ 308, 11 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 86, 3 ], "axis1": [ 320, 3 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 89, 20 ], "axis1": [ 331, 20 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 109, 68 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 385, 34 ], "axis1": [ 109, 34 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 385, 34 ], "axis1": [ 143, 34 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 109, 34 ], "axis1": [ 385, 34 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 143, 34 ], "axis1": [ 385, 34 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 177, 68 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 487, 34 ], "axis1": [ 177, 34 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 487, 34 ], "axis1": [ 211, 34 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 177, 34 ], "axis1": [ 487, 34 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 211, 34 ], "axis1": [ 487, 34 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 245, 68 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 555, 34 ], "axis1": [ 245, 34 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 555, 34 ], "axis1": [ 279, 34 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 245, 34 ], "axis1": [ 555, 34 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 279, 34 ], "axis1": [ 555, 34 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 313, 68 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 57, 2 ], "axis1": [ 313, 34 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 273, 12 ], "axis1": [ 313, 34 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 288, 20 ], "axis1": [ 313, 34 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 57, 2 ], "axis1": [ 347, 34 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 273, 12 ], "axis1": [ 347, 34 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 288, 20 ], "axis1": [ 347, 34 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 313, 2 ], "axis1": [ 57, 2 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 315, 12 ], "axis1": [ 273, 12 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 327, 20 ], "axis1": [ 288, 20 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 347, 2 ], "axis1": [ 57, 2 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 349, 12 ], "axis1": [ 273, 12 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 361, 20 ], "axis1": [ 288, 20 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 381, 6 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 592, 3 ], "axis1": [ 381, 3 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 592, 3 ], "axis1": [ 384, 3 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 381, 3 ], "axis1": [ 592, 3 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 384, 3 ], "axis1": [ 592, 3 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 387, 6 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 595, 3 ], "axis1": [ 387, 3 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 595, 3 ], "axis1": [ 390, 3 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 387, 3 ], "axis1": [ 595, 3 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 390, 3 ], "axis1": [ 595, 3 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 393, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 589, 1 ], "axis1": [ 393, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 589, 1 ], "axis1": [ 394, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 393, 1 ], "axis1": [ 589, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 394, 1 ], "axis1": [ 589, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 395, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 590, 1 ], "axis1": [ 395, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 590, 1 ], "axis1": [ 396, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 395, 1 ], "axis1": [ 590, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 396, 1 ], "axis1": [ 590, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 397, 2 ], "matrix": "L12.mlp.W_up" }, { "axis0": [ 591, 1 ], "axis1": [ 397, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 591, 1 ], "axis1": [ 398, 1 ], "matrix": "L12.mlp.W_in" }, { "axis0": [ 397, 1 ], "axis1": [ 591, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 398, 1 ], "axis1": [ 591, 1 ], "diagonal": true, "matrix": "L12.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1850, 6 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 598, 3 ], "axis1": [ 1850, 3 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 598, 3 ], "axis1": [ 1853, 3 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 1850, 3 ], "axis1": [ 598, 3 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 1853, 3 ], "axis1": [ 598, 3 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1856, 6 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 352, 3 ], "axis1": [ 1856, 3 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 352, 3 ], "axis1": [ 1859, 3 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 1856, 3 ], "axis1": [ 352, 3 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 1859, 3 ], "axis1": [ 352, 3 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1862, 6 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 264, 2 ], "axis1": [ 1862, 3 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 351, 1 ], "axis1": [ 1862, 3 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 264, 2 ], "axis1": [ 1865, 3 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 351, 1 ], "axis1": [ 1865, 3 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 1862, 2 ], "axis1": [ 264, 2 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 1864, 1 ], "axis1": [ 351, 1 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 1865, 2 ], "axis1": [ 264, 2 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 1867, 1 ], "axis1": [ 351, 1 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1868, 4 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 358, 2 ], "axis1": [ 1868, 2 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 358, 2 ], "axis1": [ 1870, 2 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 1868, 2 ], "axis1": [ 358, 2 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 1870, 2 ], "axis1": [ 358, 2 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1872, 2 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 209, 1 ], "axis1": [ 1872, 1 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 209, 1 ], "axis1": [ 1873, 1 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 1872, 1 ], "axis1": [ 209, 1 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 1873, 1 ], "axis1": [ 209, 1 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1874, 2 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 212, 1 ], "axis1": [ 1874, 1 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 212, 1 ], "axis1": [ 1875, 1 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 1874, 1 ], "axis1": [ 212, 1 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 1875, 1 ], "axis1": [ 212, 1 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1876, 2 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 256, 1 ], "axis1": [ 1876, 1 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 256, 1 ], "axis1": [ 1877, 1 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 1876, 1 ], "axis1": [ 256, 1 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 1877, 1 ], "axis1": [ 256, 1 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1878, 2 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 120, 1 ], "axis1": [ 1878, 1 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 120, 1 ], "axis1": [ 1879, 1 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 1878, 1 ], "axis1": [ 120, 1 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 1879, 1 ], "axis1": [ 120, 1 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1880, 2 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 355, 1 ], "axis1": [ 1880, 1 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 355, 1 ], "axis1": [ 1881, 1 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 1880, 1 ], "axis1": [ 355, 1 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 1881, 1 ], "axis1": [ 355, 1 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1882, 2 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 327, 1 ], "axis1": [ 1882, 1 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 327, 1 ], "axis1": [ 1883, 1 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 1882, 1 ], "axis1": [ 327, 1 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 1883, 1 ], "axis1": [ 327, 1 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1884, 2 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 207, 1 ], "axis1": [ 1884, 1 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 207, 1 ], "axis1": [ 1885, 1 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 1884, 1 ], "axis1": [ 207, 1 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 1885, 1 ], "axis1": [ 207, 1 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1886, 2 ], "matrix": "L13.mlp.W_up" }, { "axis0": [ 357, 1 ], "axis1": [ 1886, 1 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 357, 1 ], "axis1": [ 1887, 1 ], "matrix": "L13.mlp.W_in" }, { "axis0": [ 1886, 1 ], "axis1": [ 357, 1 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 1887, 1 ], "axis1": [ 357, 1 ], "diagonal": true, "matrix": "L13.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1770, 200 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 57, 1 ], "axis1": [ 1770, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 211, 1 ], "axis1": [ 1770, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 222, 1 ], "axis1": [ 1770, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 263, 1 ], "axis1": [ 1770, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 273, 12 ], "axis1": [ 1770, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 288, 16 ], "axis1": [ 1770, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 305, 2 ], "axis1": [ 1770, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 308, 11 ], "axis1": [ 1770, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 320, 3 ], "axis1": [ 1770, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 331, 11 ], "axis1": [ 1770, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 343, 2 ], "axis1": [ 1770, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 347, 1 ], "axis1": [ 1770, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 350, 1 ], "axis1": [ 1770, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 362, 27 ], "axis1": [ 1770, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 390, 9 ], "axis1": [ 1770, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 404, 1 ], "axis1": [ 1770, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 57, 1 ], "axis1": [ 1870, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 211, 1 ], "axis1": [ 1870, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 222, 1 ], "axis1": [ 1870, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 263, 1 ], "axis1": [ 1870, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 273, 12 ], "axis1": [ 1870, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 288, 16 ], "axis1": [ 1870, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 305, 2 ], "axis1": [ 1870, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 308, 11 ], "axis1": [ 1870, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 320, 3 ], "axis1": [ 1870, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 331, 11 ], "axis1": [ 1870, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 343, 2 ], "axis1": [ 1870, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 347, 1 ], "axis1": [ 1870, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 350, 1 ], "axis1": [ 1870, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 362, 27 ], "axis1": [ 1870, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 390, 9 ], "axis1": [ 1870, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 404, 1 ], "axis1": [ 1870, 100 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1770, 1 ], "axis1": [ 57, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1771, 1 ], "axis1": [ 211, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1772, 1 ], "axis1": [ 222, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1773, 1 ], "axis1": [ 263, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1774, 12 ], "axis1": [ 273, 12 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1786, 16 ], "axis1": [ 288, 16 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1802, 2 ], "axis1": [ 305, 2 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1804, 11 ], "axis1": [ 308, 11 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1815, 3 ], "axis1": [ 320, 3 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1818, 11 ], "axis1": [ 331, 11 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1829, 2 ], "axis1": [ 343, 2 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1831, 1 ], "axis1": [ 347, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1832, 1 ], "axis1": [ 350, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1833, 27 ], "axis1": [ 362, 27 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1860, 9 ], "axis1": [ 390, 9 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1869, 1 ], "axis1": [ 404, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1870, 1 ], "axis1": [ 57, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1871, 1 ], "axis1": [ 211, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1872, 1 ], "axis1": [ 222, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1873, 1 ], "axis1": [ 263, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1874, 12 ], "axis1": [ 273, 12 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1886, 16 ], "axis1": [ 288, 16 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1902, 2 ], "axis1": [ 305, 2 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1904, 11 ], "axis1": [ 308, 11 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1915, 3 ], "axis1": [ 320, 3 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1918, 11 ], "axis1": [ 331, 11 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1929, 2 ], "axis1": [ 343, 2 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1931, 1 ], "axis1": [ 347, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1932, 1 ], "axis1": [ 350, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1933, 27 ], "axis1": [ 362, 27 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1960, 9 ], "axis1": [ 390, 9 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1969, 1 ], "axis1": [ 404, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1970, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 174, 16 ], "axis1": [ 1970, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 210, 1 ], "axis1": [ 1970, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 174, 16 ], "axis1": [ 1987, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 210, 1 ], "axis1": [ 1987, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 1970, 16 ], "axis1": [ 174, 16 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1986, 1 ], "axis1": [ 210, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 1987, 16 ], "axis1": [ 174, 16 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2003, 1 ], "axis1": [ 210, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 2004, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 42, 4 ], "axis1": [ 2004, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 213, 9 ], "axis1": [ 2004, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 223, 1 ], "axis1": [ 2004, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 236, 3 ], "axis1": [ 2004, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 42, 4 ], "axis1": [ 2021, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 213, 9 ], "axis1": [ 2021, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 223, 1 ], "axis1": [ 2021, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 236, 3 ], "axis1": [ 2021, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 2004, 4 ], "axis1": [ 42, 4 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2008, 9 ], "axis1": [ 213, 9 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2017, 1 ], "axis1": [ 223, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2018, 3 ], "axis1": [ 236, 3 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2021, 4 ], "axis1": [ 42, 4 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2025, 9 ], "axis1": [ 213, 9 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2034, 1 ], "axis1": [ 223, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2035, 3 ], "axis1": [ 236, 3 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 2038, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 18, 3 ], "axis1": [ 2038, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 55, 2 ], "axis1": [ 2038, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 59, 7 ], "axis1": [ 2038, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 68, 1 ], "axis1": [ 2038, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 106, 1 ], "axis1": [ 2038, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 326, 1 ], "axis1": [ 2038, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 328, 2 ], "axis1": [ 2038, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 18, 3 ], "axis1": [ 2055, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 55, 2 ], "axis1": [ 2055, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 59, 7 ], "axis1": [ 2055, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 68, 1 ], "axis1": [ 2055, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 106, 1 ], "axis1": [ 2055, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 326, 1 ], "axis1": [ 2055, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 328, 2 ], "axis1": [ 2055, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 2038, 3 ], "axis1": [ 18, 3 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2041, 2 ], "axis1": [ 55, 2 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2043, 7 ], "axis1": [ 59, 7 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2050, 1 ], "axis1": [ 68, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2051, 1 ], "axis1": [ 106, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2052, 1 ], "axis1": [ 326, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2053, 2 ], "axis1": [ 328, 2 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2055, 3 ], "axis1": [ 18, 3 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2058, 2 ], "axis1": [ 55, 2 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2060, 7 ], "axis1": [ 59, 7 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2067, 1 ], "axis1": [ 68, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2068, 1 ], "axis1": [ 106, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2069, 1 ], "axis1": [ 326, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2070, 2 ], "axis1": [ 328, 2 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 2072, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 49, 2 ], "axis1": [ 2072, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 239, 8 ], "axis1": [ 2072, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 248, 3 ], "axis1": [ 2072, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 252, 4 ], "axis1": [ 2072, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 49, 2 ], "axis1": [ 2089, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 239, 8 ], "axis1": [ 2089, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 248, 3 ], "axis1": [ 2089, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 252, 4 ], "axis1": [ 2089, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 2072, 2 ], "axis1": [ 49, 2 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2074, 8 ], "axis1": [ 239, 8 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2082, 3 ], "axis1": [ 248, 3 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2085, 4 ], "axis1": [ 252, 4 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2089, 2 ], "axis1": [ 49, 2 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2091, 8 ], "axis1": [ 239, 8 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2099, 3 ], "axis1": [ 248, 3 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2102, 4 ], "axis1": [ 252, 4 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 2106, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 51, 4 ], "axis1": [ 2106, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 67, 1 ], "axis1": [ 2106, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 258, 5 ], "axis1": [ 2106, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 266, 7 ], "axis1": [ 2106, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 51, 4 ], "axis1": [ 2123, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 67, 1 ], "axis1": [ 2123, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 258, 5 ], "axis1": [ 2123, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 266, 7 ], "axis1": [ 2123, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 2106, 4 ], "axis1": [ 51, 4 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2110, 1 ], "axis1": [ 67, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2111, 5 ], "axis1": [ 258, 5 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2116, 7 ], "axis1": [ 266, 7 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2123, 4 ], "axis1": [ 51, 4 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2127, 1 ], "axis1": [ 67, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2128, 5 ], "axis1": [ 258, 5 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2133, 7 ], "axis1": [ 266, 7 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 2140, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 21, 5 ], "axis1": [ 2140, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 66, 1 ], "axis1": [ 2140, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 102, 4 ], "axis1": [ 2140, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 107, 4 ], "axis1": [ 2140, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 137, 2 ], "axis1": [ 2140, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 173, 1 ], "axis1": [ 2140, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 21, 5 ], "axis1": [ 2157, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 66, 1 ], "axis1": [ 2157, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 102, 4 ], "axis1": [ 2157, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 107, 4 ], "axis1": [ 2157, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 137, 2 ], "axis1": [ 2157, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 173, 1 ], "axis1": [ 2157, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 2140, 5 ], "axis1": [ 21, 5 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2145, 1 ], "axis1": [ 66, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2146, 4 ], "axis1": [ 102, 4 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2150, 4 ], "axis1": [ 107, 4 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2154, 2 ], "axis1": [ 137, 2 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2156, 1 ], "axis1": [ 173, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2157, 5 ], "axis1": [ 21, 5 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2162, 1 ], "axis1": [ 66, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2163, 4 ], "axis1": [ 102, 4 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2167, 4 ], "axis1": [ 107, 4 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2171, 2 ], "axis1": [ 137, 2 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2173, 1 ], "axis1": [ 173, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 2174, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 525, 17 ], "axis1": [ 2174, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 525, 17 ], "axis1": [ 2191, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 2174, 17 ], "axis1": [ 525, 17 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2191, 17 ], "axis1": [ 525, 17 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 2208, 34 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 346, 1 ], "axis1": [ 2208, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 506, 5 ], "axis1": [ 2208, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 512, 11 ], "axis1": [ 2208, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 346, 1 ], "axis1": [ 2225, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 506, 5 ], "axis1": [ 2225, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 512, 11 ], "axis1": [ 2225, 17 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 2208, 1 ], "axis1": [ 346, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2209, 5 ], "axis1": [ 506, 5 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2214, 11 ], "axis1": [ 512, 11 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2225, 1 ], "axis1": [ 346, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2226, 5 ], "axis1": [ 506, 5 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2231, 11 ], "axis1": [ 512, 11 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 2242, 6 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 469, 3 ], "axis1": [ 2242, 3 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 469, 3 ], "axis1": [ 2245, 3 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 2242, 3 ], "axis1": [ 469, 3 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2245, 3 ], "axis1": [ 469, 3 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 2248, 4 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 523, 2 ], "axis1": [ 2248, 2 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 523, 2 ], "axis1": [ 2250, 2 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 2248, 2 ], "axis1": [ 523, 2 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2250, 2 ], "axis1": [ 523, 2 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 2252, 2 ], "matrix": "L14.mlp.W_up" }, { "axis0": [ 58, 1 ], "axis1": [ 2252, 1 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 58, 1 ], "axis1": [ 2253, 1 ], "matrix": "L14.mlp.W_in" }, { "axis0": [ 2252, 1 ], "axis1": [ 58, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 2253, 1 ], "axis1": [ 58, 1 ], "diagonal": true, "matrix": "L14.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1054, 34 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 345, 1 ], "axis1": [ 1054, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 490, 16 ], "axis1": [ 1054, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 345, 1 ], "axis1": [ 1071, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 490, 16 ], "axis1": [ 1071, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1054, 1 ], "axis1": [ 345, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 1055, 16 ], "axis1": [ 490, 16 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 1071, 1 ], "axis1": [ 345, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 1072, 16 ], "axis1": [ 490, 16 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1088, 34 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 691, 17 ], "axis1": [ 1088, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 691, 17 ], "axis1": [ 1105, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1088, 17 ], "axis1": [ 691, 17 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 1105, 17 ], "axis1": [ 691, 17 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1122, 34 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 708, 17 ], "axis1": [ 1122, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 708, 17 ], "axis1": [ 1139, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1122, 17 ], "axis1": [ 708, 17 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 1139, 17 ], "axis1": [ 708, 17 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1156, 34 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 606, 17 ], "axis1": [ 1156, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 606, 17 ], "axis1": [ 1173, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1156, 17 ], "axis1": [ 606, 17 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 1173, 17 ], "axis1": [ 606, 17 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1190, 34 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 640, 17 ], "axis1": [ 1190, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 640, 17 ], "axis1": [ 1207, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1190, 17 ], "axis1": [ 640, 17 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 1207, 17 ], "axis1": [ 640, 17 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1224, 34 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 843, 17 ], "axis1": [ 1224, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 843, 17 ], "axis1": [ 1241, 17 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1224, 17 ], "axis1": [ 843, 17 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 1241, 17 ], "axis1": [ 843, 17 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1258, 4 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 120, 1 ], "axis1": [ 1258, 2 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 327, 1 ], "axis1": [ 1258, 2 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 120, 1 ], "axis1": [ 1260, 2 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 327, 1 ], "axis1": [ 1260, 2 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1258, 1 ], "axis1": [ 120, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 1259, 1 ], "axis1": [ 327, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 1260, 1 ], "axis1": [ 120, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 1261, 1 ], "axis1": [ 327, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1262, 4 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 860, 2 ], "axis1": [ 1262, 2 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 860, 2 ], "axis1": [ 1264, 2 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1262, 2 ], "axis1": [ 860, 2 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 1264, 2 ], "axis1": [ 860, 2 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1266, 4 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 862, 2 ], "axis1": [ 1266, 2 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 862, 2 ], "axis1": [ 1268, 2 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1266, 2 ], "axis1": [ 862, 2 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 1268, 2 ], "axis1": [ 862, 2 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1270, 2 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 33, 1 ], "axis1": [ 1270, 1 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 33, 1 ], "axis1": [ 1271, 1 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1270, 1 ], "axis1": [ 33, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 1271, 1 ], "axis1": [ 33, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1272, 2 ], "matrix": "L15.mlp.W_up" }, { "axis0": [ 725, 1 ], "axis1": [ 1272, 1 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 725, 1 ], "axis1": [ 1273, 1 ], "matrix": "L15.mlp.W_in" }, { "axis0": [ 1272, 1 ], "axis1": [ 725, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 1273, 1 ], "axis1": [ 725, 1 ], "diagonal": true, "matrix": "L15.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 522, 34 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 116, 4 ], "axis1": [ 522, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 349, 1 ], "axis1": [ 522, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 580, 4 ], "axis1": [ 522, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 585, 3 ], "axis1": [ 522, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 601, 5 ], "axis1": [ 522, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 116, 4 ], "axis1": [ 539, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 349, 1 ], "axis1": [ 539, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 580, 4 ], "axis1": [ 539, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 585, 3 ], "axis1": [ 539, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 601, 5 ], "axis1": [ 539, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 522, 4 ], "axis1": [ 116, 4 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 526, 1 ], "axis1": [ 349, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 527, 4 ], "axis1": [ 580, 4 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 531, 3 ], "axis1": [ 585, 3 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 534, 5 ], "axis1": [ 601, 5 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 539, 4 ], "axis1": [ 116, 4 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 543, 1 ], "axis1": [ 349, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 544, 4 ], "axis1": [ 580, 4 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 548, 3 ], "axis1": [ 585, 3 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 551, 5 ], "axis1": [ 601, 5 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 556, 34 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 623, 17 ], "axis1": [ 556, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 623, 17 ], "axis1": [ 573, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 556, 17 ], "axis1": [ 623, 17 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 573, 17 ], "axis1": [ 623, 17 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 590, 34 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 290, 14 ], "axis1": [ 590, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 305, 2 ], "axis1": [ 590, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 393, 1 ], "axis1": [ 590, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 290, 14 ], "axis1": [ 607, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 305, 2 ], "axis1": [ 607, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 393, 1 ], "axis1": [ 607, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 590, 14 ], "axis1": [ 290, 14 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 604, 2 ], "axis1": [ 305, 2 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 606, 1 ], "axis1": [ 393, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 607, 14 ], "axis1": [ 290, 14 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 621, 2 ], "axis1": [ 305, 2 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 623, 1 ], "axis1": [ 393, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 624, 34 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 177, 2 ], "axis1": [ 624, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 180, 9 ], "axis1": [ 624, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 210, 1 ], "axis1": [ 624, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 214, 1 ], "axis1": [ 624, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 217, 1 ], "axis1": [ 624, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 333, 1 ], "axis1": [ 624, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 392, 1 ], "axis1": [ 624, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 404, 1 ], "axis1": [ 624, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 177, 2 ], "axis1": [ 641, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 180, 9 ], "axis1": [ 641, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 210, 1 ], "axis1": [ 641, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 214, 1 ], "axis1": [ 641, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 217, 1 ], "axis1": [ 641, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 333, 1 ], "axis1": [ 641, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 392, 1 ], "axis1": [ 641, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 404, 1 ], "axis1": [ 641, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 624, 2 ], "axis1": [ 177, 2 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 626, 9 ], "axis1": [ 180, 9 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 635, 1 ], "axis1": [ 210, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 636, 1 ], "axis1": [ 214, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 637, 1 ], "axis1": [ 217, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 638, 1 ], "axis1": [ 333, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 639, 1 ], "axis1": [ 392, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 640, 1 ], "axis1": [ 404, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 641, 2 ], "axis1": [ 177, 2 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 643, 9 ], "axis1": [ 180, 9 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 652, 1 ], "axis1": [ 210, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 653, 1 ], "axis1": [ 214, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 654, 1 ], "axis1": [ 217, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 655, 1 ], "axis1": [ 333, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 656, 1 ], "axis1": [ 392, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 657, 1 ], "axis1": [ 404, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 658, 34 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 52, 3 ], "axis1": [ 658, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 56, 1 ], "axis1": [ 658, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 58, 6 ], "axis1": [ 658, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 65, 1 ], "axis1": [ 658, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 173, 4 ], "axis1": [ 658, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 179, 1 ], "axis1": [ 658, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 326, 1 ], "axis1": [ 658, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 52, 3 ], "axis1": [ 675, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 56, 1 ], "axis1": [ 675, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 58, 6 ], "axis1": [ 675, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 65, 1 ], "axis1": [ 675, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 173, 4 ], "axis1": [ 675, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 179, 1 ], "axis1": [ 675, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 326, 1 ], "axis1": [ 675, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 658, 3 ], "axis1": [ 52, 3 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 661, 1 ], "axis1": [ 56, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 662, 6 ], "axis1": [ 58, 6 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 668, 1 ], "axis1": [ 65, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 669, 4 ], "axis1": [ 173, 4 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 673, 1 ], "axis1": [ 179, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 674, 1 ], "axis1": [ 326, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 675, 3 ], "axis1": [ 52, 3 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 678, 1 ], "axis1": [ 56, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 679, 6 ], "axis1": [ 58, 6 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 685, 1 ], "axis1": [ 65, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 686, 4 ], "axis1": [ 173, 4 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 690, 1 ], "axis1": [ 179, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 691, 1 ], "axis1": [ 326, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 692, 34 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 64, 1 ], "axis1": [ 692, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 308, 11 ], "axis1": [ 692, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 320, 3 ], "axis1": [ 692, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 328, 1 ], "axis1": [ 692, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 338, 1 ], "axis1": [ 692, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 64, 1 ], "axis1": [ 709, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 308, 11 ], "axis1": [ 709, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 320, 3 ], "axis1": [ 709, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 328, 1 ], "axis1": [ 709, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 338, 1 ], "axis1": [ 709, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 692, 1 ], "axis1": [ 64, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 693, 11 ], "axis1": [ 308, 11 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 704, 3 ], "axis1": [ 320, 3 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 707, 1 ], "axis1": [ 328, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 708, 1 ], "axis1": [ 338, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 709, 1 ], "axis1": [ 64, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 710, 11 ], "axis1": [ 308, 11 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 721, 3 ], "axis1": [ 320, 3 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 724, 1 ], "axis1": [ 328, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 725, 1 ], "axis1": [ 338, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 726, 34 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 207, 1 ], "axis1": [ 726, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 209, 1 ], "axis1": [ 726, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 212, 1 ], "axis1": [ 726, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 256, 1 ], "axis1": [ 726, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 264, 2 ], "axis1": [ 726, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 351, 5 ], "axis1": [ 726, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 357, 5 ], "axis1": [ 726, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 542, 1 ], "axis1": [ 726, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 207, 1 ], "axis1": [ 743, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 209, 1 ], "axis1": [ 743, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 212, 1 ], "axis1": [ 743, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 256, 1 ], "axis1": [ 743, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 264, 2 ], "axis1": [ 743, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 351, 5 ], "axis1": [ 743, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 357, 5 ], "axis1": [ 743, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 542, 1 ], "axis1": [ 743, 17 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 726, 1 ], "axis1": [ 207, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 727, 1 ], "axis1": [ 209, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 728, 1 ], "axis1": [ 212, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 729, 1 ], "axis1": [ 256, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 730, 2 ], "axis1": [ 264, 2 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 732, 5 ], "axis1": [ 351, 5 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 737, 5 ], "axis1": [ 357, 5 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 742, 1 ], "axis1": [ 542, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 743, 1 ], "axis1": [ 207, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 744, 1 ], "axis1": [ 209, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 745, 1 ], "axis1": [ 212, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 746, 1 ], "axis1": [ 256, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 747, 2 ], "axis1": [ 264, 2 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 749, 5 ], "axis1": [ 351, 5 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 754, 5 ], "axis1": [ 357, 5 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 759, 1 ], "axis1": [ 542, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 760, 6 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 45, 1 ], "axis1": [ 760, 3 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 223, 1 ], "axis1": [ 760, 3 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 239, 1 ], "axis1": [ 760, 3 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 45, 1 ], "axis1": [ 763, 3 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 223, 1 ], "axis1": [ 763, 3 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 239, 1 ], "axis1": [ 763, 3 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 760, 1 ], "axis1": [ 45, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 761, 1 ], "axis1": [ 223, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 762, 1 ], "axis1": [ 239, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 763, 1 ], "axis1": [ 45, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 764, 1 ], "axis1": [ 223, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 765, 1 ], "axis1": [ 239, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 766, 6 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 236, 3 ], "axis1": [ 766, 3 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 236, 3 ], "axis1": [ 769, 3 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 766, 3 ], "axis1": [ 236, 3 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 769, 3 ], "axis1": [ 236, 3 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 772, 6 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 42, 1 ], "axis1": [ 772, 3 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 211, 1 ], "axis1": [ 772, 3 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 213, 1 ], "axis1": [ 772, 3 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 42, 1 ], "axis1": [ 775, 3 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 211, 1 ], "axis1": [ 775, 3 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 213, 1 ], "axis1": [ 775, 3 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 772, 1 ], "axis1": [ 42, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 773, 1 ], "axis1": [ 211, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 774, 1 ], "axis1": [ 213, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 775, 1 ], "axis1": [ 42, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 776, 1 ], "axis1": [ 211, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 777, 1 ], "axis1": [ 213, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 778, 6 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 215, 2 ], "axis1": [ 778, 3 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 334, 1 ], "axis1": [ 778, 3 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 215, 2 ], "axis1": [ 781, 3 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 334, 1 ], "axis1": [ 781, 3 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 778, 2 ], "axis1": [ 215, 2 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 780, 1 ], "axis1": [ 334, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 781, 2 ], "axis1": [ 215, 2 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 783, 1 ], "axis1": [ 334, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 784, 6 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 115, 1 ], "axis1": [ 784, 3 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 577, 2 ], "axis1": [ 784, 3 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 115, 1 ], "axis1": [ 787, 3 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 577, 2 ], "axis1": [ 787, 3 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 784, 1 ], "axis1": [ 115, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 785, 2 ], "axis1": [ 577, 2 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 787, 1 ], "axis1": [ 115, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 788, 2 ], "axis1": [ 577, 2 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 790, 2 ], "matrix": "L16.mlp.W_up" }, { "axis0": [ 208, 1 ], "axis1": [ 790, 1 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 208, 1 ], "axis1": [ 791, 1 ], "matrix": "L16.mlp.W_in" }, { "axis0": [ 790, 1 ], "axis1": [ 208, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 791, 1 ], "axis1": [ 208, 1 ], "diagonal": true, "matrix": "L16.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 462, 34 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 657, 17 ], "axis1": [ 462, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 657, 17 ], "axis1": [ 479, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 462, 17 ], "axis1": [ 657, 17 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 479, 17 ], "axis1": [ 657, 17 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 496, 34 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 674, 17 ], "axis1": [ 496, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 674, 17 ], "axis1": [ 513, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 496, 17 ], "axis1": [ 674, 17 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 513, 17 ], "axis1": [ 674, 17 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 530, 34 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 18, 2 ], "axis1": [ 530, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 329, 1 ], "axis1": [ 530, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 344, 2 ], "axis1": [ 530, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 468, 4 ], "axis1": [ 530, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 490, 8 ], "axis1": [ 530, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 18, 2 ], "axis1": [ 547, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 329, 1 ], "axis1": [ 547, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 344, 2 ], "axis1": [ 547, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 468, 4 ], "axis1": [ 547, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 490, 8 ], "axis1": [ 547, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 530, 2 ], "axis1": [ 18, 2 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 532, 1 ], "axis1": [ 329, 1 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 533, 2 ], "axis1": [ 344, 2 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 535, 4 ], "axis1": [ 468, 4 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 539, 8 ], "axis1": [ 490, 8 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 547, 2 ], "axis1": [ 18, 2 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 549, 1 ], "axis1": [ 329, 1 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 550, 2 ], "axis1": [ 344, 2 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 552, 4 ], "axis1": [ 468, 4 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 556, 8 ], "axis1": [ 490, 8 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 564, 34 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 20, 1 ], "axis1": [ 564, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 102, 1 ], "axis1": [ 564, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 346, 1 ], "axis1": [ 564, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 498, 4 ], "axis1": [ 564, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 505, 6 ], "axis1": [ 564, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 512, 3 ], "axis1": [ 564, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 519, 1 ], "axis1": [ 564, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 20, 1 ], "axis1": [ 581, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 102, 1 ], "axis1": [ 581, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 346, 1 ], "axis1": [ 581, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 498, 4 ], "axis1": [ 581, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 505, 6 ], "axis1": [ 581, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 512, 3 ], "axis1": [ 581, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 519, 1 ], "axis1": [ 581, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 564, 1 ], "axis1": [ 20, 1 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 565, 1 ], "axis1": [ 102, 1 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 566, 1 ], "axis1": [ 346, 1 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 567, 4 ], "axis1": [ 498, 4 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 571, 6 ], "axis1": [ 505, 6 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 577, 3 ], "axis1": [ 512, 3 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 580, 1 ], "axis1": [ 519, 1 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 581, 1 ], "axis1": [ 20, 1 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 582, 1 ], "axis1": [ 102, 1 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 583, 1 ], "axis1": [ 346, 1 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 584, 4 ], "axis1": [ 498, 4 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 588, 6 ], "axis1": [ 505, 6 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 594, 3 ], "axis1": [ 512, 3 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 597, 1 ], "axis1": [ 519, 1 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 598, 34 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 111, 1 ], "axis1": [ 598, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 543, 16 ], "axis1": [ 598, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 111, 1 ], "axis1": [ 615, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 543, 16 ], "axis1": [ 615, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 598, 1 ], "axis1": [ 111, 1 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 599, 16 ], "axis1": [ 543, 16 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 615, 1 ], "axis1": [ 111, 1 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 616, 16 ], "axis1": [ 543, 16 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 632, 34 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 342, 1 ], "axis1": [ 632, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 399, 5 ], "axis1": [ 632, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 405, 9 ], "axis1": [ 632, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 415, 2 ], "axis1": [ 632, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 342, 1 ], "axis1": [ 649, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 399, 5 ], "axis1": [ 649, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 405, 9 ], "axis1": [ 649, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 415, 2 ], "axis1": [ 649, 17 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 632, 1 ], "axis1": [ 342, 1 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 633, 5 ], "axis1": [ 399, 5 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 638, 9 ], "axis1": [ 405, 9 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 647, 2 ], "axis1": [ 415, 2 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 649, 1 ], "axis1": [ 342, 1 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 650, 5 ], "axis1": [ 399, 5 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 655, 9 ], "axis1": [ 405, 9 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 664, 2 ], "axis1": [ 415, 2 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 666, 6 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 374, 3 ], "axis1": [ 666, 3 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 374, 3 ], "axis1": [ 669, 3 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 666, 3 ], "axis1": [ 374, 3 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 669, 3 ], "axis1": [ 374, 3 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 672, 6 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 368, 3 ], "axis1": [ 672, 3 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 368, 3 ], "axis1": [ 675, 3 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 672, 3 ], "axis1": [ 368, 3 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 675, 3 ], "axis1": [ 368, 3 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 678, 6 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 340, 1 ], "axis1": [ 678, 3 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 362, 2 ], "axis1": [ 678, 3 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 340, 1 ], "axis1": [ 681, 3 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 362, 2 ], "axis1": [ 681, 3 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 678, 1 ], "axis1": [ 340, 1 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 679, 2 ], "axis1": [ 362, 2 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 681, 1 ], "axis1": [ 340, 1 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 682, 2 ], "axis1": [ 362, 2 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 684, 6 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 371, 3 ], "axis1": [ 684, 3 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 371, 3 ], "axis1": [ 687, 3 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 684, 3 ], "axis1": [ 371, 3 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 687, 3 ], "axis1": [ 371, 3 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 690, 6 ], "matrix": "L17.mlp.W_up" }, { "axis0": [ 365, 3 ], "axis1": [ 690, 3 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 365, 3 ], "axis1": [ 693, 3 ], "matrix": "L17.mlp.W_in" }, { "axis0": [ 690, 3 ], "axis1": [ 365, 3 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 693, 3 ], "axis1": [ 365, 3 ], "diagonal": true, "matrix": "L17.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 444, 34 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 42, 1 ], "axis1": [ 444, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 45, 1 ], "axis1": [ 444, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 210, 1 ], "axis1": [ 444, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 212, 8 ], "axis1": [ 444, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 223, 1 ], "axis1": [ 444, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 236, 4 ], "axis1": [ 444, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 256, 1 ], "axis1": [ 444, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 42, 1 ], "axis1": [ 461, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 45, 1 ], "axis1": [ 461, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 210, 1 ], "axis1": [ 461, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 212, 8 ], "axis1": [ 461, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 223, 1 ], "axis1": [ 461, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 236, 4 ], "axis1": [ 461, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 256, 1 ], "axis1": [ 461, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 444, 1 ], "axis1": [ 42, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 445, 1 ], "axis1": [ 45, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 446, 1 ], "axis1": [ 210, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 447, 8 ], "axis1": [ 212, 8 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 455, 1 ], "axis1": [ 223, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 456, 4 ], "axis1": [ 236, 4 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 460, 1 ], "axis1": [ 256, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 461, 1 ], "axis1": [ 42, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 462, 1 ], "axis1": [ 45, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 463, 1 ], "axis1": [ 210, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 464, 8 ], "axis1": [ 212, 8 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 472, 1 ], "axis1": [ 223, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 473, 4 ], "axis1": [ 236, 4 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 477, 1 ], "axis1": [ 256, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 478, 34 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 52, 3 ], "axis1": [ 478, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 56, 1 ], "axis1": [ 478, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 58, 2 ], "axis1": [ 478, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 264, 2 ], "axis1": [ 478, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 290, 9 ], "axis1": [ 478, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 52, 3 ], "axis1": [ 495, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 56, 1 ], "axis1": [ 495, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 58, 2 ], "axis1": [ 495, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 264, 2 ], "axis1": [ 495, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 290, 9 ], "axis1": [ 495, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 478, 3 ], "axis1": [ 52, 3 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 481, 1 ], "axis1": [ 56, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 482, 2 ], "axis1": [ 58, 2 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 484, 2 ], "axis1": [ 264, 2 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 486, 9 ], "axis1": [ 290, 9 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 495, 3 ], "axis1": [ 52, 3 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 498, 1 ], "axis1": [ 56, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 499, 2 ], "axis1": [ 58, 2 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 501, 2 ], "axis1": [ 264, 2 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 503, 9 ], "axis1": [ 290, 9 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 512, 34 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 63, 2 ], "axis1": [ 512, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 68, 1 ], "axis1": [ 512, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 316, 1 ], "axis1": [ 512, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 318, 1 ], "axis1": [ 512, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 320, 3 ], "axis1": [ 512, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 328, 1 ], "axis1": [ 512, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 351, 1 ], "axis1": [ 512, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 354, 1 ], "axis1": [ 512, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 356, 6 ], "axis1": [ 512, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 63, 2 ], "axis1": [ 529, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 68, 1 ], "axis1": [ 529, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 316, 1 ], "axis1": [ 529, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 318, 1 ], "axis1": [ 529, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 320, 3 ], "axis1": [ 529, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 328, 1 ], "axis1": [ 529, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 351, 1 ], "axis1": [ 529, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 354, 1 ], "axis1": [ 529, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 356, 6 ], "axis1": [ 529, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 512, 2 ], "axis1": [ 63, 2 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 514, 1 ], "axis1": [ 68, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 515, 1 ], "axis1": [ 316, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 516, 1 ], "axis1": [ 318, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 517, 3 ], "axis1": [ 320, 3 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 520, 1 ], "axis1": [ 328, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 521, 1 ], "axis1": [ 351, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 522, 1 ], "axis1": [ 354, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 523, 6 ], "axis1": [ 356, 6 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 529, 2 ], "axis1": [ 63, 2 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 531, 1 ], "axis1": [ 68, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 532, 1 ], "axis1": [ 316, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 533, 1 ], "axis1": [ 318, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 534, 3 ], "axis1": [ 320, 3 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 537, 1 ], "axis1": [ 328, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 538, 1 ], "axis1": [ 351, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 539, 1 ], "axis1": [ 354, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 540, 6 ], "axis1": [ 356, 6 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 546, 34 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 417, 17 ], "axis1": [ 546, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 417, 17 ], "axis1": [ 563, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 546, 17 ], "axis1": [ 417, 17 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 563, 17 ], "axis1": [ 417, 17 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 580, 34 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 434, 17 ], "axis1": [ 580, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 434, 17 ], "axis1": [ 597, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 580, 17 ], "axis1": [ 434, 17 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 597, 17 ], "axis1": [ 434, 17 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 614, 34 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 451, 17 ], "axis1": [ 614, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 451, 17 ], "axis1": [ 631, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 614, 17 ], "axis1": [ 451, 17 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 631, 17 ], "axis1": [ 451, 17 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 648, 34 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 112, 3 ], "axis1": [ 648, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 348, 1 ], "axis1": [ 648, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 560, 3 ], "axis1": [ 648, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 565, 4 ], "axis1": [ 648, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 570, 4 ], "axis1": [ 648, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 575, 2 ], "axis1": [ 648, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 112, 3 ], "axis1": [ 665, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 348, 1 ], "axis1": [ 665, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 560, 3 ], "axis1": [ 665, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 565, 4 ], "axis1": [ 665, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 570, 4 ], "axis1": [ 665, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 575, 2 ], "axis1": [ 665, 17 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 648, 3 ], "axis1": [ 112, 3 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 651, 1 ], "axis1": [ 348, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 652, 3 ], "axis1": [ 560, 3 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 655, 4 ], "axis1": [ 565, 4 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 659, 4 ], "axis1": [ 570, 4 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 663, 2 ], "axis1": [ 575, 2 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 665, 3 ], "axis1": [ 112, 3 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 668, 1 ], "axis1": [ 348, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 669, 3 ], "axis1": [ 560, 3 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 672, 4 ], "axis1": [ 565, 4 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 676, 4 ], "axis1": [ 570, 4 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 680, 2 ], "axis1": [ 575, 2 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 682, 6 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 44, 1 ], "axis1": [ 682, 3 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 220, 1 ], "axis1": [ 682, 3 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 222, 1 ], "axis1": [ 682, 3 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 44, 1 ], "axis1": [ 685, 3 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 220, 1 ], "axis1": [ 685, 3 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 222, 1 ], "axis1": [ 685, 3 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 682, 1 ], "axis1": [ 44, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 683, 1 ], "axis1": [ 220, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 684, 1 ], "axis1": [ 222, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 685, 1 ], "axis1": [ 44, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 686, 1 ], "axis1": [ 220, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 687, 1 ], "axis1": [ 222, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 688, 6 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 208, 1 ], "axis1": [ 688, 3 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 334, 1 ], "axis1": [ 688, 3 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 404, 1 ], "axis1": [ 688, 3 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 208, 1 ], "axis1": [ 691, 3 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 334, 1 ], "axis1": [ 691, 3 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 404, 1 ], "axis1": [ 691, 3 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 688, 1 ], "axis1": [ 208, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 689, 1 ], "axis1": [ 334, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 690, 1 ], "axis1": [ 404, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 691, 1 ], "axis1": [ 208, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 692, 1 ], "axis1": [ 334, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 693, 1 ], "axis1": [ 404, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 694, 6 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 207, 1 ], "axis1": [ 694, 3 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 209, 1 ], "axis1": [ 694, 3 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 211, 1 ], "axis1": [ 694, 3 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 207, 1 ], "axis1": [ 697, 3 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 209, 1 ], "axis1": [ 697, 3 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 211, 1 ], "axis1": [ 697, 3 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 694, 1 ], "axis1": [ 207, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 695, 1 ], "axis1": [ 209, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 696, 1 ], "axis1": [ 211, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 697, 1 ], "axis1": [ 207, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 698, 1 ], "axis1": [ 209, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 699, 1 ], "axis1": [ 211, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 700, 6 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 188, 1 ], "axis1": [ 700, 3 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 327, 1 ], "axis1": [ 700, 3 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 333, 1 ], "axis1": [ 700, 3 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 188, 1 ], "axis1": [ 703, 3 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 327, 1 ], "axis1": [ 703, 3 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 333, 1 ], "axis1": [ 703, 3 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 700, 1 ], "axis1": [ 188, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 701, 1 ], "axis1": [ 327, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 702, 1 ], "axis1": [ 333, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 703, 1 ], "axis1": [ 188, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 704, 1 ], "axis1": [ 327, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 705, 1 ], "axis1": [ 333, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 706, 2 ], "matrix": "L18.mlp.W_up" }, { "axis0": [ 35, 1 ], "axis1": [ 706, 1 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 35, 1 ], "axis1": [ 707, 1 ], "matrix": "L18.mlp.W_in" }, { "axis0": [ 706, 1 ], "axis1": [ 35, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 707, 1 ], "axis1": [ 35, 1 ], "diagonal": true, "matrix": "L18.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 236, 34 ], "matrix": "L19.mlp.W_up" }, { "axis0": [ 60, 3 ], "axis1": [ 236, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 303, 1 ], "axis1": [ 236, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 305, 2 ], "axis1": [ 236, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 308, 4 ], "axis1": [ 236, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 313, 1 ], "axis1": [ 236, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 315, 1 ], "axis1": [ 236, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 338, 1 ], "axis1": [ 236, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 340, 1 ], "axis1": [ 236, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 362, 2 ], "axis1": [ 236, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 393, 1 ], "axis1": [ 236, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 60, 3 ], "axis1": [ 253, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 303, 1 ], "axis1": [ 253, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 305, 2 ], "axis1": [ 253, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 308, 4 ], "axis1": [ 253, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 313, 1 ], "axis1": [ 253, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 315, 1 ], "axis1": [ 253, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 338, 1 ], "axis1": [ 253, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 340, 1 ], "axis1": [ 253, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 362, 2 ], "axis1": [ 253, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 393, 1 ], "axis1": [ 253, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 236, 3 ], "axis1": [ 60, 3 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 239, 1 ], "axis1": [ 303, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 240, 2 ], "axis1": [ 305, 2 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 242, 4 ], "axis1": [ 308, 4 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 246, 1 ], "axis1": [ 313, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 247, 1 ], "axis1": [ 315, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 248, 1 ], "axis1": [ 338, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 249, 1 ], "axis1": [ 340, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 250, 2 ], "axis1": [ 362, 2 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 252, 1 ], "axis1": [ 393, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 253, 3 ], "axis1": [ 60, 3 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 256, 1 ], "axis1": [ 303, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 257, 2 ], "axis1": [ 305, 2 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 259, 4 ], "axis1": [ 308, 4 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 263, 1 ], "axis1": [ 313, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 264, 1 ], "axis1": [ 315, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 265, 1 ], "axis1": [ 338, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 266, 1 ], "axis1": [ 340, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 267, 2 ], "axis1": [ 362, 2 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 269, 1 ], "axis1": [ 393, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 270, 34 ], "matrix": "L19.mlp.W_up" }, { "axis0": [ 365, 12 ], "axis1": [ 270, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 399, 5 ], "axis1": [ 270, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 365, 12 ], "axis1": [ 287, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 399, 5 ], "axis1": [ 287, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 270, 12 ], "axis1": [ 365, 12 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 282, 5 ], "axis1": [ 399, 5 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 287, 12 ], "axis1": [ 365, 12 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 299, 5 ], "axis1": [ 399, 5 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 304, 34 ], "matrix": "L19.mlp.W_up" }, { "axis0": [ 18, 1 ], "axis1": [ 304, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 329, 1 ], "axis1": [ 304, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 342, 1 ], "axis1": [ 304, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 344, 1 ], "axis1": [ 304, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 405, 9 ], "axis1": [ 304, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 415, 2 ], "axis1": [ 304, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 468, 2 ], "axis1": [ 304, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 18, 1 ], "axis1": [ 321, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 329, 1 ], "axis1": [ 321, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 342, 1 ], "axis1": [ 321, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 344, 1 ], "axis1": [ 321, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 405, 9 ], "axis1": [ 321, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 415, 2 ], "axis1": [ 321, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 468, 2 ], "axis1": [ 321, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 304, 1 ], "axis1": [ 18, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 305, 1 ], "axis1": [ 329, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 306, 1 ], "axis1": [ 342, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 307, 1 ], "axis1": [ 344, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 308, 9 ], "axis1": [ 405, 9 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 317, 2 ], "axis1": [ 415, 2 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 319, 2 ], "axis1": [ 468, 2 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 321, 1 ], "axis1": [ 18, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 322, 1 ], "axis1": [ 329, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 323, 1 ], "axis1": [ 342, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 324, 1 ], "axis1": [ 344, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 325, 9 ], "axis1": [ 405, 9 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 334, 2 ], "axis1": [ 415, 2 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 336, 2 ], "axis1": [ 468, 2 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 338, 34 ], "matrix": "L19.mlp.W_up" }, { "axis0": [ 19, 1 ], "axis1": [ 338, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 345, 1 ], "axis1": [ 338, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 470, 2 ], "axis1": [ 338, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 490, 12 ], "axis1": [ 338, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 510, 1 ], "axis1": [ 338, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 19, 1 ], "axis1": [ 355, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 345, 1 ], "axis1": [ 355, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 470, 2 ], "axis1": [ 355, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 490, 12 ], "axis1": [ 355, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 510, 1 ], "axis1": [ 355, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 338, 1 ], "axis1": [ 19, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 339, 1 ], "axis1": [ 345, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 340, 2 ], "axis1": [ 470, 2 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 342, 12 ], "axis1": [ 490, 12 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 354, 1 ], "axis1": [ 510, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 355, 1 ], "axis1": [ 19, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 356, 1 ], "axis1": [ 345, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 357, 2 ], "axis1": [ 470, 2 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 359, 12 ], "axis1": [ 490, 12 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 371, 1 ], "axis1": [ 510, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 372, 34 ], "matrix": "L19.mlp.W_up" }, { "axis0": [ 341, 1 ], "axis1": [ 372, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 343, 1 ], "axis1": [ 372, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 377, 12 ], "axis1": [ 372, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 390, 1 ], "axis1": [ 372, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 394, 1 ], "axis1": [ 372, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 398, 1 ], "axis1": [ 372, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 341, 1 ], "axis1": [ 389, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 343, 1 ], "axis1": [ 389, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 377, 12 ], "axis1": [ 389, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 390, 1 ], "axis1": [ 389, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 394, 1 ], "axis1": [ 389, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 398, 1 ], "axis1": [ 389, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 372, 1 ], "axis1": [ 341, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 373, 1 ], "axis1": [ 343, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 374, 12 ], "axis1": [ 377, 12 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 386, 1 ], "axis1": [ 390, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 387, 1 ], "axis1": [ 394, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 388, 1 ], "axis1": [ 398, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 389, 1 ], "axis1": [ 341, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 390, 1 ], "axis1": [ 343, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 391, 12 ], "axis1": [ 377, 12 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 403, 1 ], "axis1": [ 390, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 404, 1 ], "axis1": [ 394, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 405, 1 ], "axis1": [ 398, 1 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 406, 34 ], "matrix": "L19.mlp.W_up" }, { "axis0": [ 472, 17 ], "axis1": [ 406, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 472, 17 ], "axis1": [ 423, 17 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 406, 17 ], "axis1": [ 472, 17 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 423, 17 ], "axis1": [ 472, 17 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 440, 6 ], "matrix": "L19.mlp.W_up" }, { "axis0": [ 299, 3 ], "axis1": [ 440, 3 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 299, 3 ], "axis1": [ 443, 3 ], "matrix": "L19.mlp.W_in" }, { "axis0": [ 440, 3 ], "axis1": [ 299, 3 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 443, 3 ], "axis1": [ 299, 3 ], "diagonal": true, "matrix": "L19.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 22, 34 ], "matrix": "L20.mlp.W_up" }, { "axis0": [ 44, 2 ], "axis1": [ 22, 17 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 51, 1 ], "axis1": [ 22, 17 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 216, 5 ], "axis1": [ 22, 17 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 222, 2 ], "axis1": [ 22, 17 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 236, 4 ], "axis1": [ 22, 17 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 256, 1 ], "axis1": [ 22, 17 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 262, 2 ], "axis1": [ 22, 17 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 44, 2 ], "axis1": [ 39, 17 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 51, 1 ], "axis1": [ 39, 17 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 216, 5 ], "axis1": [ 39, 17 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 222, 2 ], "axis1": [ 39, 17 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 236, 4 ], "axis1": [ 39, 17 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 256, 1 ], "axis1": [ 39, 17 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 262, 2 ], "axis1": [ 39, 17 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 22, 2 ], "axis1": [ 44, 2 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 24, 1 ], "axis1": [ 51, 1 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 25, 5 ], "axis1": [ 216, 5 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 30, 2 ], "axis1": [ 222, 2 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 32, 4 ], "axis1": [ 236, 4 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 36, 1 ], "axis1": [ 256, 1 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 37, 2 ], "axis1": [ 262, 2 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 39, 2 ], "axis1": [ 44, 2 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 41, 1 ], "axis1": [ 51, 1 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 42, 5 ], "axis1": [ 216, 5 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 47, 2 ], "axis1": [ 222, 2 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 49, 4 ], "axis1": [ 236, 4 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 53, 1 ], "axis1": [ 256, 1 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 54, 2 ], "axis1": [ 262, 2 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 56, 34 ], "matrix": "L20.mlp.W_up" }, { "axis0": [ 52, 4 ], "axis1": [ 56, 17 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 67, 1 ], "axis1": [ 56, 17 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 258, 4 ], "axis1": [ 56, 17 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 264, 3 ], "axis1": [ 56, 17 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 268, 3 ], "axis1": [ 56, 17 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 273, 1 ], "axis1": [ 56, 17 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 391, 1 ], "axis1": [ 56, 17 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 52, 4 ], "axis1": [ 73, 17 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 67, 1 ], "axis1": [ 73, 17 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 258, 4 ], "axis1": [ 73, 17 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 264, 3 ], "axis1": [ 73, 17 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 268, 3 ], "axis1": [ 73, 17 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 273, 1 ], "axis1": [ 73, 17 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 391, 1 ], "axis1": [ 73, 17 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 56, 4 ], "axis1": [ 52, 4 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 60, 1 ], "axis1": [ 67, 1 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 61, 4 ], "axis1": [ 258, 4 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 65, 3 ], "axis1": [ 264, 3 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 68, 3 ], "axis1": [ 268, 3 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 71, 1 ], "axis1": [ 273, 1 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 72, 1 ], "axis1": [ 391, 1 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 73, 4 ], "axis1": [ 52, 4 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 77, 1 ], "axis1": [ 67, 1 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 78, 4 ], "axis1": [ 258, 4 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 82, 3 ], "axis1": [ 264, 3 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 85, 3 ], "axis1": [ 268, 3 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 88, 1 ], "axis1": [ 273, 1 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 89, 1 ], "axis1": [ 391, 1 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 90, 34 ], "matrix": "L20.mlp.W_up" }, { "axis0": [ 46, 1 ], "axis1": [ 90, 17 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 198, 9 ], "axis1": [ 90, 17 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 224, 7 ], "axis1": [ 90, 17 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 46, 1 ], "axis1": [ 107, 17 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 198, 9 ], "axis1": [ 107, 17 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 224, 7 ], "axis1": [ 107, 17 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 90, 1 ], "axis1": [ 46, 1 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 91, 9 ], "axis1": [ 198, 9 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 100, 7 ], "axis1": [ 224, 7 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 107, 1 ], "axis1": [ 46, 1 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 108, 9 ], "axis1": [ 198, 9 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 117, 7 ], "axis1": [ 224, 7 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 124, 2 ], "matrix": "L20.mlp.W_up" }, { "axis0": [ 40, 1 ], "axis1": [ 124, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 40, 1 ], "axis1": [ 125, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 124, 1 ], "axis1": [ 40, 1 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 125, 1 ], "axis1": [ 40, 1 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 126, 2 ], "matrix": "L20.mlp.W_up" }, { "axis0": [ 321, 1 ], "axis1": [ 126, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 321, 1 ], "axis1": [ 127, 1 ], "matrix": "L20.mlp.W_in" }, { "axis0": [ 126, 1 ], "axis1": [ 321, 1 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 127, 1 ], "axis1": [ 321, 1 ], "diagonal": true, "matrix": "L20.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 7, 34 ], "matrix": "L21.mlp.W_up" }, { "axis0": [ 139, 17 ], "axis1": [ 7, 17 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 139, 17 ], "axis1": [ 24, 17 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 7, 17 ], "axis1": [ 139, 17 ], "diagonal": true, "matrix": "L21.mlp.W_out" }, { "axis0": [ 24, 17 ], "axis1": [ 139, 17 ], "diagonal": true, "matrix": "L21.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 41, 34 ], "matrix": "L21.mlp.W_up" }, { "axis0": [ 28, 1 ], "axis1": [ 41, 17 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 121, 16 ], "axis1": [ 41, 17 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 28, 1 ], "axis1": [ 58, 17 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 121, 16 ], "axis1": [ 58, 17 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 41, 1 ], "axis1": [ 28, 1 ], "diagonal": true, "matrix": "L21.mlp.W_out" }, { "axis0": [ 42, 16 ], "axis1": [ 121, 16 ], "diagonal": true, "matrix": "L21.mlp.W_out" }, { "axis0": [ 58, 1 ], "axis1": [ 28, 1 ], "diagonal": true, "matrix": "L21.mlp.W_out" }, { "axis0": [ 59, 16 ], "axis1": [ 121, 16 ], "diagonal": true, "matrix": "L21.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 75, 34 ], "matrix": "L21.mlp.W_up" }, { "axis0": [ 39, 1 ], "axis1": [ 75, 17 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 49, 1 ], "axis1": [ 75, 17 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 194, 3 ], "axis1": [ 75, 17 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 240, 7 ], "axis1": [ 75, 17 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 248, 3 ], "axis1": [ 75, 17 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 252, 1 ], "axis1": [ 75, 17 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 335, 1 ], "axis1": [ 75, 17 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 39, 1 ], "axis1": [ 92, 17 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 49, 1 ], "axis1": [ 92, 17 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 194, 3 ], "axis1": [ 92, 17 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 240, 7 ], "axis1": [ 92, 17 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 248, 3 ], "axis1": [ 92, 17 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 252, 1 ], "axis1": [ 92, 17 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 335, 1 ], "axis1": [ 92, 17 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 75, 1 ], "axis1": [ 39, 1 ], "diagonal": true, "matrix": "L21.mlp.W_out" }, { "axis0": [ 76, 1 ], "axis1": [ 49, 1 ], "diagonal": true, "matrix": "L21.mlp.W_out" }, { "axis0": [ 77, 3 ], "axis1": [ 194, 3 ], "diagonal": true, "matrix": "L21.mlp.W_out" }, { "axis0": [ 80, 7 ], "axis1": [ 240, 7 ], "diagonal": true, "matrix": "L21.mlp.W_out" }, { "axis0": [ 87, 3 ], "axis1": [ 248, 3 ], "diagonal": true, "matrix": "L21.mlp.W_out" }, { "axis0": [ 90, 1 ], "axis1": [ 252, 1 ], "diagonal": true, "matrix": "L21.mlp.W_out" }, { "axis0": [ 91, 1 ], "axis1": [ 335, 1 ], "diagonal": true, "matrix": "L21.mlp.W_out" }, { "axis0": [ 92, 1 ], "axis1": [ 39, 1 ], "diagonal": true, "matrix": "L21.mlp.W_out" }, { "axis0": [ 93, 1 ], "axis1": [ 49, 1 ], "diagonal": true, "matrix": "L21.mlp.W_out" }, { "axis0": [ 94, 3 ], "axis1": [ 194, 3 ], "diagonal": true, "matrix": "L21.mlp.W_out" }, { "axis0": [ 97, 7 ], "axis1": [ 240, 7 ], "diagonal": true, "matrix": "L21.mlp.W_out" }, { "axis0": [ 104, 3 ], "axis1": [ 248, 3 ], "diagonal": true, "matrix": "L21.mlp.W_out" }, { "axis0": [ 107, 1 ], "axis1": [ 252, 1 ], "diagonal": true, "matrix": "L21.mlp.W_out" }, { "axis0": [ 108, 1 ], "axis1": [ 335, 1 ], "diagonal": true, "matrix": "L21.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 109, 2 ], "matrix": "L21.mlp.W_up" }, { "axis0": [ 50, 1 ], "axis1": [ 109, 1 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 50, 1 ], "axis1": [ 110, 1 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 109, 1 ], "axis1": [ 50, 1 ], "diagonal": true, "matrix": "L21.mlp.W_out" }, { "axis0": [ 110, 1 ], "axis1": [ 50, 1 ], "diagonal": true, "matrix": "L21.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 111, 2 ], "matrix": "L21.mlp.W_up" }, { "axis0": [ 253, 1 ], "axis1": [ 111, 1 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 253, 1 ], "axis1": [ 112, 1 ], "matrix": "L21.mlp.W_in" }, { "axis0": [ 111, 1 ], "axis1": [ 253, 1 ], "diagonal": true, "matrix": "L21.mlp.W_out" }, { "axis0": [ 112, 1 ], "axis1": [ 253, 1 ], "diagonal": true, "matrix": "L21.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 13, 2 ], "matrix": "L22.mlp.W_up" }, { "axis0": [ 223, 1 ], "axis1": [ 13, 1 ], "matrix": "L22.mlp.W_in" }, { "axis0": [ 223, 1 ], "axis1": [ 14, 1 ], "matrix": "L22.mlp.W_in" }, { "axis0": [ 13, 1 ], "axis1": [ 223, 1 ], "diagonal": true, "matrix": "L22.mlp.W_out" }, { "axis0": [ 14, 1 ], "axis1": [ 223, 1 ], "diagonal": true, "matrix": "L22.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 15, 2 ], "matrix": "L22.mlp.W_up" }, { "axis0": [ 225, 1 ], "axis1": [ 15, 1 ], "matrix": "L22.mlp.W_in" }, { "axis0": [ 225, 1 ], "axis1": [ 16, 1 ], "matrix": "L22.mlp.W_in" }, { "axis0": [ 15, 1 ], "axis1": [ 225, 1 ], "diagonal": true, "matrix": "L22.mlp.W_out" }, { "axis0": [ 16, 1 ], "axis1": [ 225, 1 ], "diagonal": true, "matrix": "L22.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 17, 2 ], "matrix": "L22.mlp.W_up" }, { "axis0": [ 226, 1 ], "axis1": [ 17, 1 ], "matrix": "L22.mlp.W_in" }, { "axis0": [ 226, 1 ], "axis1": [ 18, 1 ], "matrix": "L22.mlp.W_in" }, { "axis0": [ 17, 1 ], "axis1": [ 226, 1 ], "diagonal": true, "matrix": "L22.mlp.W_out" }, { "axis0": [ 18, 1 ], "axis1": [ 226, 1 ], "diagonal": true, "matrix": "L22.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 19, 2 ], "matrix": "L22.mlp.W_up" }, { "axis0": [ 254, 1 ], "axis1": [ 19, 1 ], "matrix": "L22.mlp.W_in" }, { "axis0": [ 254, 1 ], "axis1": [ 20, 1 ], "matrix": "L22.mlp.W_in" }, { "axis0": [ 19, 1 ], "axis1": [ 254, 1 ], "diagonal": true, "matrix": "L22.mlp.W_out" }, { "axis0": [ 20, 1 ], "axis1": [ 254, 1 ], "diagonal": true, "matrix": "L22.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 21, 2 ], "matrix": "L22.mlp.W_up" }, { "axis0": [ 224, 1 ], "axis1": [ 21, 1 ], "matrix": "L22.mlp.W_in" }, { "axis0": [ 224, 1 ], "axis1": [ 22, 1 ], "matrix": "L22.mlp.W_in" }, { "axis0": [ 21, 1 ], "axis1": [ 224, 1 ], "diagonal": true, "matrix": "L22.mlp.W_out" }, { "axis0": [ 22, 1 ], "axis1": [ 224, 1 ], "diagonal": true, "matrix": "L22.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 5, 2 ], "matrix": "L23.mlp.W_up" }, { "axis0": [ 125, 1 ], "axis1": [ 5, 1 ], "matrix": "L23.mlp.W_in" }, { "axis0": [ 125, 1 ], "axis1": [ 6, 1 ], "matrix": "L23.mlp.W_in" }, { "axis0": [ 5, 1 ], "axis1": [ 125, 1 ], "diagonal": true, "matrix": "L23.mlp.W_out" }, { "axis0": [ 6, 1 ], "axis1": [ 125, 1 ], "diagonal": true, "matrix": "L23.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 7, 2 ], "matrix": "L23.mlp.W_up" }, { "axis0": [ 126, 1 ], "axis1": [ 7, 1 ], "matrix": "L23.mlp.W_in" }, { "axis0": [ 126, 1 ], "axis1": [ 8, 1 ], "matrix": "L23.mlp.W_in" }, { "axis0": [ 7, 1 ], "axis1": [ 126, 1 ], "diagonal": true, "matrix": "L23.mlp.W_out" }, { "axis0": [ 8, 1 ], "axis1": [ 126, 1 ], "diagonal": true, "matrix": "L23.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 3, 2 ], "matrix": "L24.mlp.W_up" }, { "axis0": [ 127, 1 ], "axis1": [ 3, 1 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 127, 1 ], "axis1": [ 4, 1 ], "matrix": "L24.mlp.W_in" }, { "axis0": [ 3, 1 ], "axis1": [ 127, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 4, 1 ], "axis1": [ 127, 1 ], "diagonal": true, "matrix": "L24.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1633, 34 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 40, 1 ], "axis1": [ 1633, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 44, 1 ], "axis1": [ 1633, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 198, 9 ], "axis1": [ 1633, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 216, 5 ], "axis1": [ 1633, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 222, 1 ], "axis1": [ 1633, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 40, 1 ], "axis1": [ 1650, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 44, 1 ], "axis1": [ 1650, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 198, 9 ], "axis1": [ 1650, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 216, 5 ], "axis1": [ 1650, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 222, 1 ], "axis1": [ 1650, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 1633, 1 ], "axis1": [ 40, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1634, 1 ], "axis1": [ 44, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1635, 9 ], "axis1": [ 198, 9 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1644, 5 ], "axis1": [ 216, 5 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1649, 1 ], "axis1": [ 222, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1650, 1 ], "axis1": [ 40, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1651, 1 ], "axis1": [ 44, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1652, 9 ], "axis1": [ 198, 9 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1661, 5 ], "axis1": [ 216, 5 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1666, 1 ], "axis1": [ 222, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1667, 34 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 18, 1 ], "axis1": [ 1667, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 29, 4 ], "axis1": [ 1667, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 34, 1 ], "axis1": [ 1667, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 65, 1 ], "axis1": [ 1667, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 156, 8 ], "axis1": [ 1667, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 326, 1 ], "axis1": [ 1667, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 329, 1 ], "axis1": [ 1667, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 18, 1 ], "axis1": [ 1684, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 29, 4 ], "axis1": [ 1684, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 34, 1 ], "axis1": [ 1684, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 65, 1 ], "axis1": [ 1684, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 156, 8 ], "axis1": [ 1684, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 326, 1 ], "axis1": [ 1684, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 329, 1 ], "axis1": [ 1684, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 1667, 1 ], "axis1": [ 18, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1668, 4 ], "axis1": [ 29, 4 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1672, 1 ], "axis1": [ 34, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1673, 1 ], "axis1": [ 65, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1674, 8 ], "axis1": [ 156, 8 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1682, 1 ], "axis1": [ 326, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1683, 1 ], "axis1": [ 329, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1684, 1 ], "axis1": [ 18, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1685, 4 ], "axis1": [ 29, 4 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1689, 1 ], "axis1": [ 34, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1690, 1 ], "axis1": [ 65, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1691, 8 ], "axis1": [ 156, 8 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1699, 1 ], "axis1": [ 326, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1700, 1 ], "axis1": [ 329, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1701, 34 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 36, 3 ], "axis1": [ 1701, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 164, 9 ], "axis1": [ 1701, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 190, 4 ], "axis1": [ 1701, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 197, 1 ], "axis1": [ 1701, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 36, 3 ], "axis1": [ 1718, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 164, 9 ], "axis1": [ 1718, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 190, 4 ], "axis1": [ 1718, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 197, 1 ], "axis1": [ 1718, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 1701, 3 ], "axis1": [ 36, 3 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1704, 9 ], "axis1": [ 164, 9 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1713, 4 ], "axis1": [ 190, 4 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1717, 1 ], "axis1": [ 197, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1718, 3 ], "axis1": [ 36, 3 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1721, 9 ], "axis1": [ 164, 9 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1730, 4 ], "axis1": [ 190, 4 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1734, 1 ], "axis1": [ 197, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1735, 34 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 35, 1 ], "axis1": [ 1735, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 42, 2 ], "axis1": [ 1735, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 188, 1 ], "axis1": [ 1735, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 207, 9 ], "axis1": [ 1735, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 327, 1 ], "axis1": [ 1735, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 333, 2 ], "axis1": [ 1735, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 404, 1 ], "axis1": [ 1735, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 35, 1 ], "axis1": [ 1752, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 42, 2 ], "axis1": [ 1752, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 188, 1 ], "axis1": [ 1752, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 207, 9 ], "axis1": [ 1752, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 327, 1 ], "axis1": [ 1752, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 333, 2 ], "axis1": [ 1752, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 404, 1 ], "axis1": [ 1752, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 1735, 1 ], "axis1": [ 35, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1736, 2 ], "axis1": [ 42, 2 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1738, 1 ], "axis1": [ 188, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1739, 9 ], "axis1": [ 207, 9 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1748, 1 ], "axis1": [ 327, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1749, 2 ], "axis1": [ 333, 2 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1751, 1 ], "axis1": [ 404, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1752, 1 ], "axis1": [ 35, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1753, 2 ], "axis1": [ 42, 2 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1755, 1 ], "axis1": [ 188, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1756, 9 ], "axis1": [ 207, 9 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1765, 1 ], "axis1": [ 327, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1766, 2 ], "axis1": [ 333, 2 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1768, 1 ], "axis1": [ 404, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1769, 34 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 57, 1 ], "axis1": [ 1769, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 274, 11 ], "axis1": [ 1769, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 288, 2 ], "axis1": [ 1769, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 302, 1 ], "axis1": [ 1769, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 337, 1 ], "axis1": [ 1769, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 339, 1 ], "axis1": [ 1769, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 57, 1 ], "axis1": [ 1786, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 274, 11 ], "axis1": [ 1786, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 288, 2 ], "axis1": [ 1786, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 302, 1 ], "axis1": [ 1786, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 337, 1 ], "axis1": [ 1786, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 339, 1 ], "axis1": [ 1786, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 1769, 1 ], "axis1": [ 57, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1770, 11 ], "axis1": [ 274, 11 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1781, 2 ], "axis1": [ 288, 2 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1783, 1 ], "axis1": [ 302, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1784, 1 ], "axis1": [ 337, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1785, 1 ], "axis1": [ 339, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1786, 1 ], "axis1": [ 57, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1787, 11 ], "axis1": [ 274, 11 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1798, 2 ], "axis1": [ 288, 2 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1800, 1 ], "axis1": [ 302, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1801, 1 ], "axis1": [ 337, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1802, 1 ], "axis1": [ 339, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1803, 34 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 33, 1 ], "axis1": [ 1803, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 173, 15 ], "axis1": [ 1803, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 392, 1 ], "axis1": [ 1803, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 33, 1 ], "axis1": [ 1820, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 173, 15 ], "axis1": [ 1820, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 392, 1 ], "axis1": [ 1820, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 1803, 1 ], "axis1": [ 33, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1804, 15 ], "axis1": [ 173, 15 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1819, 1 ], "axis1": [ 392, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1820, 1 ], "axis1": [ 33, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1821, 15 ], "axis1": [ 173, 15 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1836, 1 ], "axis1": [ 392, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1837, 34 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 27, 1 ], "axis1": [ 1837, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 69, 16 ], "axis1": [ 1837, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 27, 1 ], "axis1": [ 1854, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 69, 16 ], "axis1": [ 1854, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 1837, 1 ], "axis1": [ 27, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1838, 16 ], "axis1": [ 69, 16 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1854, 1 ], "axis1": [ 27, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1855, 16 ], "axis1": [ 69, 16 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1871, 34 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 85, 17 ], "axis1": [ 1871, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 85, 17 ], "axis1": [ 1888, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 1871, 17 ], "axis1": [ 85, 17 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1888, 17 ], "axis1": [ 85, 17 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1905, 34 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 28, 1 ], "axis1": [ 1905, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 134, 3 ], "axis1": [ 1905, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 139, 13 ], "axis1": [ 1905, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 28, 1 ], "axis1": [ 1922, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 134, 3 ], "axis1": [ 1922, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 139, 13 ], "axis1": [ 1922, 17 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 1905, 1 ], "axis1": [ 28, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1906, 3 ], "axis1": [ 134, 3 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1909, 13 ], "axis1": [ 139, 13 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1922, 1 ], "axis1": [ 28, 1 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1923, 3 ], "axis1": [ 134, 3 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1926, 13 ], "axis1": [ 139, 13 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1939, 24 ], "matrix": "L26.mlp.W_up" }, { "axis0": [ 122, 12 ], "axis1": [ 1939, 12 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 122, 12 ], "axis1": [ 1951, 12 ], "matrix": "L26.mlp.W_in" }, { "axis0": [ 1939, 12 ], "axis1": [ 122, 12 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 1951, 12 ], "axis1": [ 122, 12 ], "diagonal": true, "matrix": "L26.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 137, 408 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 39, 1 ], "axis1": [ 137, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 45, 2 ], "axis1": [ 137, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 49, 8 ], "axis1": [ 137, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 58, 7 ], "axis1": [ 137, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 67, 2 ], "axis1": [ 137, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 121, 1 ], "axis1": [ 137, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 152, 4 ], "axis1": [ 137, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 194, 3 ], "axis1": [ 137, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 223, 4 ], "axis1": [ 137, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 228, 3 ], "axis1": [ 137, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 236, 11 ], "axis1": [ 137, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 248, 3 ], "axis1": [ 137, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 252, 2 ], "axis1": [ 137, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 255, 2 ], "axis1": [ 137, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 258, 6 ], "axis1": [ 137, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 265, 2 ], "axis1": [ 137, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 268, 3 ], "axis1": [ 137, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 273, 1 ], "axis1": [ 137, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 290, 12 ], "axis1": [ 137, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 303, 1 ], "axis1": [ 137, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 305, 2 ], "axis1": [ 137, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 308, 4 ], "axis1": [ 137, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 313, 1 ], "axis1": [ 137, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 315, 2 ], "axis1": [ 137, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 318, 1 ], "axis1": [ 137, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 320, 3 ], "axis1": [ 137, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 328, 1 ], "axis1": [ 137, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 335, 2 ], "axis1": [ 137, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 338, 1 ], "axis1": [ 137, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 340, 5 ], "axis1": [ 137, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 351, 1 ], "axis1": [ 137, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 354, 1 ], "axis1": [ 137, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 356, 27 ], "axis1": [ 137, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 384, 5 ], "axis1": [ 137, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 390, 2 ], "axis1": [ 137, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 393, 2 ], "axis1": [ 137, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 398, 6 ], "axis1": [ 137, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 405, 9 ], "axis1": [ 137, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 415, 51 ], "axis1": [ 137, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 39, 1 ], "axis1": [ 341, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 45, 2 ], "axis1": [ 341, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 49, 8 ], "axis1": [ 341, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 58, 7 ], "axis1": [ 341, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 67, 2 ], "axis1": [ 341, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 121, 1 ], "axis1": [ 341, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 152, 4 ], "axis1": [ 341, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 194, 3 ], "axis1": [ 341, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 223, 4 ], "axis1": [ 341, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 228, 3 ], "axis1": [ 341, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 236, 11 ], "axis1": [ 341, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 248, 3 ], "axis1": [ 341, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 252, 2 ], "axis1": [ 341, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 255, 2 ], "axis1": [ 341, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 258, 6 ], "axis1": [ 341, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 265, 2 ], "axis1": [ 341, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 268, 3 ], "axis1": [ 341, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 273, 1 ], "axis1": [ 341, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 290, 12 ], "axis1": [ 341, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 303, 1 ], "axis1": [ 341, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 305, 2 ], "axis1": [ 341, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 308, 4 ], "axis1": [ 341, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 313, 1 ], "axis1": [ 341, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 315, 2 ], "axis1": [ 341, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 318, 1 ], "axis1": [ 341, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 320, 3 ], "axis1": [ 341, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 328, 1 ], "axis1": [ 341, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 335, 2 ], "axis1": [ 341, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 338, 1 ], "axis1": [ 341, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 340, 5 ], "axis1": [ 341, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 351, 1 ], "axis1": [ 341, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 354, 1 ], "axis1": [ 341, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 356, 27 ], "axis1": [ 341, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 384, 5 ], "axis1": [ 341, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 390, 2 ], "axis1": [ 341, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 393, 2 ], "axis1": [ 341, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 398, 6 ], "axis1": [ 341, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 405, 9 ], "axis1": [ 341, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 415, 51 ], "axis1": [ 341, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 137, 1 ], "axis1": [ 39, 1 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 138, 2 ], "axis1": [ 45, 2 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 140, 8 ], "axis1": [ 49, 8 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 148, 7 ], "axis1": [ 58, 7 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 155, 2 ], "axis1": [ 67, 2 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 157, 1 ], "axis1": [ 121, 1 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 158, 4 ], "axis1": [ 152, 4 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 162, 3 ], "axis1": [ 194, 3 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 165, 4 ], "axis1": [ 223, 4 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 169, 3 ], "axis1": [ 228, 3 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 172, 11 ], "axis1": [ 236, 11 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 183, 3 ], "axis1": [ 248, 3 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 186, 2 ], "axis1": [ 252, 2 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 188, 2 ], "axis1": [ 255, 2 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 190, 6 ], "axis1": [ 258, 6 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 196, 2 ], "axis1": [ 265, 2 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 198, 3 ], "axis1": [ 268, 3 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 201, 1 ], "axis1": [ 273, 1 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 202, 12 ], "axis1": [ 290, 12 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 214, 1 ], "axis1": [ 303, 1 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 215, 2 ], "axis1": [ 305, 2 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 217, 4 ], "axis1": [ 308, 4 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 221, 1 ], "axis1": [ 313, 1 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 222, 2 ], "axis1": [ 315, 2 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 224, 1 ], "axis1": [ 318, 1 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 225, 3 ], "axis1": [ 320, 3 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 228, 1 ], "axis1": [ 328, 1 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 229, 2 ], "axis1": [ 335, 2 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 231, 1 ], "axis1": [ 338, 1 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 232, 5 ], "axis1": [ 340, 5 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 237, 1 ], "axis1": [ 351, 1 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 238, 1 ], "axis1": [ 354, 1 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 239, 27 ], "axis1": [ 356, 27 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 266, 5 ], "axis1": [ 384, 5 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 271, 2 ], "axis1": [ 390, 2 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 273, 2 ], "axis1": [ 393, 2 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 275, 6 ], "axis1": [ 398, 6 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 281, 9 ], "axis1": [ 405, 9 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 290, 51 ], "axis1": [ 415, 51 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 341, 1 ], "axis1": [ 39, 1 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 342, 2 ], "axis1": [ 45, 2 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 344, 8 ], "axis1": [ 49, 8 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 352, 7 ], "axis1": [ 58, 7 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 359, 2 ], "axis1": [ 67, 2 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 361, 1 ], "axis1": [ 121, 1 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 362, 4 ], "axis1": [ 152, 4 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 366, 3 ], "axis1": [ 194, 3 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 369, 4 ], "axis1": [ 223, 4 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 373, 3 ], "axis1": [ 228, 3 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 376, 11 ], "axis1": [ 236, 11 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 387, 3 ], "axis1": [ 248, 3 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 390, 2 ], "axis1": [ 252, 2 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 392, 2 ], "axis1": [ 255, 2 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 394, 6 ], "axis1": [ 258, 6 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 400, 2 ], "axis1": [ 265, 2 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 402, 3 ], "axis1": [ 268, 3 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 405, 1 ], "axis1": [ 273, 1 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 406, 12 ], "axis1": [ 290, 12 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 418, 1 ], "axis1": [ 303, 1 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 419, 2 ], "axis1": [ 305, 2 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 421, 4 ], "axis1": [ 308, 4 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 425, 1 ], "axis1": [ 313, 1 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 426, 2 ], "axis1": [ 315, 2 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 428, 1 ], "axis1": [ 318, 1 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 429, 3 ], "axis1": [ 320, 3 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 432, 1 ], "axis1": [ 328, 1 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 433, 2 ], "axis1": [ 335, 2 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 435, 1 ], "axis1": [ 338, 1 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 436, 5 ], "axis1": [ 340, 5 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 441, 1 ], "axis1": [ 351, 1 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 442, 1 ], "axis1": [ 354, 1 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 443, 27 ], "axis1": [ 356, 27 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 470, 5 ], "axis1": [ 384, 5 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 475, 2 ], "axis1": [ 390, 2 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 477, 2 ], "axis1": [ 393, 2 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 479, 6 ], "axis1": [ 398, 6 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 485, 9 ], "axis1": [ 405, 9 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 494, 51 ], "axis1": [ 415, 51 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 545, 408 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 19, 7 ], "axis1": [ 545, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 102, 19 ], "axis1": [ 545, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 345, 6 ], "axis1": [ 545, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 352, 1 ], "axis1": [ 545, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 395, 1 ], "axis1": [ 545, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 466, 13 ], "axis1": [ 545, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 480, 9 ], "axis1": [ 545, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 490, 12 ], "axis1": [ 545, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 505, 6 ], "axis1": [ 545, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 512, 10 ], "axis1": [ 545, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 525, 9 ], "axis1": [ 545, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 535, 3 ], "axis1": [ 545, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 540, 3 ], "axis1": [ 545, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 544, 5 ], "axis1": [ 545, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 550, 3 ], "axis1": [ 545, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 555, 1 ], "axis1": [ 545, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 557, 2 ], "axis1": [ 545, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 560, 3 ], "axis1": [ 545, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 565, 4 ], "axis1": [ 545, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 570, 4 ], "axis1": [ 545, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 576, 3 ], "axis1": [ 545, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 580, 4 ], "axis1": [ 545, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 585, 3 ], "axis1": [ 545, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 601, 30 ], "axis1": [ 545, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 632, 37 ], "axis1": [ 545, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 670, 6 ], "axis1": [ 545, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 19, 7 ], "axis1": [ 749, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 102, 19 ], "axis1": [ 749, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 345, 6 ], "axis1": [ 749, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 352, 1 ], "axis1": [ 749, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 395, 1 ], "axis1": [ 749, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 466, 13 ], "axis1": [ 749, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 480, 9 ], "axis1": [ 749, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 490, 12 ], "axis1": [ 749, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 505, 6 ], "axis1": [ 749, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 512, 10 ], "axis1": [ 749, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 525, 9 ], "axis1": [ 749, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 535, 3 ], "axis1": [ 749, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 540, 3 ], "axis1": [ 749, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 544, 5 ], "axis1": [ 749, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 550, 3 ], "axis1": [ 749, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 555, 1 ], "axis1": [ 749, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 557, 2 ], "axis1": [ 749, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 560, 3 ], "axis1": [ 749, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 565, 4 ], "axis1": [ 749, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 570, 4 ], "axis1": [ 749, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 576, 3 ], "axis1": [ 749, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 580, 4 ], "axis1": [ 749, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 585, 3 ], "axis1": [ 749, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 601, 30 ], "axis1": [ 749, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 632, 37 ], "axis1": [ 749, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 670, 6 ], "axis1": [ 749, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 545, 7 ], "axis1": [ 19, 7 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 552, 19 ], "axis1": [ 102, 19 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 571, 6 ], "axis1": [ 345, 6 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 577, 1 ], "axis1": [ 352, 1 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 578, 1 ], "axis1": [ 395, 1 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 579, 13 ], "axis1": [ 466, 13 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 592, 9 ], "axis1": [ 480, 9 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 601, 12 ], "axis1": [ 490, 12 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 613, 6 ], "axis1": [ 505, 6 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 619, 10 ], "axis1": [ 512, 10 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 629, 9 ], "axis1": [ 525, 9 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 638, 3 ], "axis1": [ 535, 3 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 641, 3 ], "axis1": [ 540, 3 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 644, 5 ], "axis1": [ 544, 5 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 649, 3 ], "axis1": [ 550, 3 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 652, 1 ], "axis1": [ 555, 1 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 653, 2 ], "axis1": [ 557, 2 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 655, 3 ], "axis1": [ 560, 3 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 658, 4 ], "axis1": [ 565, 4 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 662, 4 ], "axis1": [ 570, 4 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 666, 3 ], "axis1": [ 576, 3 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 669, 4 ], "axis1": [ 580, 4 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 673, 3 ], "axis1": [ 585, 3 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 676, 30 ], "axis1": [ 601, 30 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 706, 37 ], "axis1": [ 632, 37 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 743, 6 ], "axis1": [ 670, 6 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 749, 7 ], "axis1": [ 19, 7 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 756, 19 ], "axis1": [ 102, 19 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 775, 6 ], "axis1": [ 345, 6 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 781, 1 ], "axis1": [ 352, 1 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 782, 1 ], "axis1": [ 395, 1 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 783, 13 ], "axis1": [ 466, 13 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 796, 9 ], "axis1": [ 480, 9 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 805, 12 ], "axis1": [ 490, 12 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 817, 6 ], "axis1": [ 505, 6 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 823, 10 ], "axis1": [ 512, 10 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 833, 9 ], "axis1": [ 525, 9 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 842, 3 ], "axis1": [ 535, 3 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 845, 3 ], "axis1": [ 540, 3 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 848, 5 ], "axis1": [ 544, 5 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 853, 3 ], "axis1": [ 550, 3 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 856, 1 ], "axis1": [ 555, 1 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 857, 2 ], "axis1": [ 557, 2 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 859, 3 ], "axis1": [ 560, 3 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 862, 4 ], "axis1": [ 565, 4 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 866, 4 ], "axis1": [ 570, 4 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 870, 3 ], "axis1": [ 576, 3 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 873, 4 ], "axis1": [ 580, 4 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 877, 3 ], "axis1": [ 585, 3 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 880, 30 ], "axis1": [ 601, 30 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 910, 37 ], "axis1": [ 632, 37 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 947, 6 ], "axis1": [ 670, 6 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 953, 408 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 66, 1 ], "axis1": [ 953, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 137, 2 ], "axis1": [ 953, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 331, 2 ], "axis1": [ 953, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 353, 1 ], "axis1": [ 953, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 355, 1 ], "axis1": [ 953, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 396, 2 ], "axis1": [ 953, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 676, 3 ], "axis1": [ 953, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 680, 6 ], "axis1": [ 953, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 691, 44 ], "axis1": [ 953, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 736, 68 ], "axis1": [ 953, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 805, 16 ], "axis1": [ 953, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 822, 33 ], "axis1": [ 953, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 864, 25 ], "axis1": [ 953, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 66, 1 ], "axis1": [ 1157, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 137, 2 ], "axis1": [ 1157, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 331, 2 ], "axis1": [ 1157, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 353, 1 ], "axis1": [ 1157, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 355, 1 ], "axis1": [ 1157, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 396, 2 ], "axis1": [ 1157, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 676, 3 ], "axis1": [ 1157, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 680, 6 ], "axis1": [ 1157, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 691, 44 ], "axis1": [ 1157, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 736, 68 ], "axis1": [ 1157, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 805, 16 ], "axis1": [ 1157, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 822, 33 ], "axis1": [ 1157, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 864, 25 ], "axis1": [ 1157, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 953, 1 ], "axis1": [ 66, 1 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 954, 2 ], "axis1": [ 137, 2 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 956, 2 ], "axis1": [ 331, 2 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 958, 1 ], "axis1": [ 353, 1 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 959, 1 ], "axis1": [ 355, 1 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 960, 2 ], "axis1": [ 396, 2 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 962, 3 ], "axis1": [ 676, 3 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 965, 6 ], "axis1": [ 680, 6 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 971, 44 ], "axis1": [ 691, 44 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 1015, 68 ], "axis1": [ 736, 68 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 1083, 16 ], "axis1": [ 805, 16 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 1099, 33 ], "axis1": [ 822, 33 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 1132, 25 ], "axis1": [ 864, 25 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 1157, 1 ], "axis1": [ 66, 1 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 1158, 2 ], "axis1": [ 137, 2 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 1160, 2 ], "axis1": [ 331, 2 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 1162, 1 ], "axis1": [ 353, 1 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 1163, 1 ], "axis1": [ 355, 1 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 1164, 2 ], "axis1": [ 396, 2 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 1166, 3 ], "axis1": [ 676, 3 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 1169, 6 ], "axis1": [ 680, 6 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 1175, 44 ], "axis1": [ 691, 44 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 1219, 68 ], "axis1": [ 736, 68 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 1287, 16 ], "axis1": [ 805, 16 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 1303, 33 ], "axis1": [ 822, 33 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 1336, 25 ], "axis1": [ 864, 25 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1361, 408 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 889, 204 ], "axis1": [ 1361, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 889, 204 ], "axis1": [ 1565, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 1361, 204 ], "axis1": [ 889, 204 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 1565, 204 ], "axis1": [ 889, 204 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 1769, 408 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 1093, 204 ], "axis1": [ 1769, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 1093, 204 ], "axis1": [ 1973, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 1769, 204 ], "axis1": [ 1093, 204 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 1973, 204 ], "axis1": [ 1093, 204 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 2177, 408 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 1297, 204 ], "axis1": [ 2177, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 1297, 204 ], "axis1": [ 2381, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 2177, 204 ], "axis1": [ 1297, 204 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 2381, 204 ], "axis1": [ 1297, 204 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 2585, 408 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 1501, 204 ], "axis1": [ 2585, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 1501, 204 ], "axis1": [ 2789, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 2585, 204 ], "axis1": [ 1501, 204 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 2789, 204 ], "axis1": [ 1501, 204 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 2993, 408 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 1705, 204 ], "axis1": [ 2993, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 1705, 204 ], "axis1": [ 3197, 204 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 2993, 204 ], "axis1": [ 1705, 204 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 3197, 204 ], "axis1": [ 1705, 204 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 3401, 2 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 285, 1 ], "axis1": [ 3401, 1 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 285, 1 ], "axis1": [ 3402, 1 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 3401, 1 ], "axis1": [ 285, 1 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 3402, 1 ], "axis1": [ 285, 1 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 3403, 2 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 286, 1 ], "axis1": [ 3403, 1 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 286, 1 ], "axis1": [ 3404, 1 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 3403, 1 ], "axis1": [ 286, 1 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 3404, 1 ], "axis1": [ 286, 1 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 3405, 2 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 231, 1 ], "axis1": [ 3405, 1 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 231, 1 ], "axis1": [ 3406, 1 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 3405, 1 ], "axis1": [ 231, 1 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 3406, 1 ], "axis1": [ 231, 1 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 3407, 2 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 233, 1 ], "axis1": [ 3407, 1 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 233, 1 ], "axis1": [ 3408, 1 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 3407, 1 ], "axis1": [ 233, 1 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 3408, 1 ], "axis1": [ 233, 1 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 3409, 2 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 287, 1 ], "axis1": [ 3409, 1 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 287, 1 ], "axis1": [ 3410, 1 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 3409, 1 ], "axis1": [ 287, 1 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 3410, 1 ], "axis1": [ 287, 1 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 3411, 2 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 234, 1 ], "axis1": [ 3411, 1 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 234, 1 ], "axis1": [ 3412, 1 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 3411, 1 ], "axis1": [ 234, 1 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 3412, 1 ], "axis1": [ 234, 1 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 3413, 2 ], "matrix": "L27.mlp.W_up" }, { "axis0": [ 235, 1 ], "axis1": [ 3413, 1 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 235, 1 ], "axis1": [ 3414, 1 ], "matrix": "L27.mlp.W_in" }, { "axis0": [ 3413, 1 ], "axis1": [ 235, 1 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 3414, 1 ], "axis1": [ 235, 1 ], "diagonal": true, "matrix": "L27.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 35, 34 ], "matrix": "L28.mlp.W_up" }, { "axis0": [ 29, 3 ], "axis1": [ 35, 17 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 71, 1 ], "axis1": [ 35, 17 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 145, 7 ], "axis1": [ 35, 17 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 156, 6 ], "axis1": [ 35, 17 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 29, 3 ], "axis1": [ 52, 17 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 71, 1 ], "axis1": [ 52, 17 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 145, 7 ], "axis1": [ 52, 17 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 156, 6 ], "axis1": [ 52, 17 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 35, 3 ], "axis1": [ 29, 3 ], "diagonal": true, "matrix": "L28.mlp.W_out" }, { "axis0": [ 38, 1 ], "axis1": [ 71, 1 ], "diagonal": true, "matrix": "L28.mlp.W_out" }, { "axis0": [ 39, 7 ], "axis1": [ 145, 7 ], "diagonal": true, "matrix": "L28.mlp.W_out" }, { "axis0": [ 46, 6 ], "axis1": [ 156, 6 ], "diagonal": true, "matrix": "L28.mlp.W_out" }, { "axis0": [ 52, 3 ], "axis1": [ 29, 3 ], "diagonal": true, "matrix": "L28.mlp.W_out" }, { "axis0": [ 55, 1 ], "axis1": [ 71, 1 ], "diagonal": true, "matrix": "L28.mlp.W_out" }, { "axis0": [ 56, 7 ], "axis1": [ 145, 7 ], "diagonal": true, "matrix": "L28.mlp.W_out" }, { "axis0": [ 63, 6 ], "axis1": [ 156, 6 ], "diagonal": true, "matrix": "L28.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 69, 34 ], "matrix": "L28.mlp.W_up" }, { "axis0": [ 44, 1 ], "axis1": [ 69, 17 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 74, 3 ], "axis1": [ 69, 17 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 80, 1 ], "axis1": [ 69, 17 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 217, 4 ], "axis1": [ 69, 17 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 222, 1 ], "axis1": [ 69, 17 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 274, 7 ], "axis1": [ 69, 17 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 44, 1 ], "axis1": [ 86, 17 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 74, 3 ], "axis1": [ 86, 17 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 80, 1 ], "axis1": [ 86, 17 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 217, 4 ], "axis1": [ 86, 17 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 222, 1 ], "axis1": [ 86, 17 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 274, 7 ], "axis1": [ 86, 17 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 69, 1 ], "axis1": [ 44, 1 ], "diagonal": true, "matrix": "L28.mlp.W_out" }, { "axis0": [ 70, 3 ], "axis1": [ 74, 3 ], "diagonal": true, "matrix": "L28.mlp.W_out" }, { "axis0": [ 73, 1 ], "axis1": [ 80, 1 ], "diagonal": true, "matrix": "L28.mlp.W_out" }, { "axis0": [ 74, 4 ], "axis1": [ 217, 4 ], "diagonal": true, "matrix": "L28.mlp.W_out" }, { "axis0": [ 78, 1 ], "axis1": [ 222, 1 ], "diagonal": true, "matrix": "L28.mlp.W_out" }, { "axis0": [ 79, 7 ], "axis1": [ 274, 7 ], "diagonal": true, "matrix": "L28.mlp.W_out" }, { "axis0": [ 86, 1 ], "axis1": [ 44, 1 ], "diagonal": true, "matrix": "L28.mlp.W_out" }, { "axis0": [ 87, 3 ], "axis1": [ 74, 3 ], "diagonal": true, "matrix": "L28.mlp.W_out" }, { "axis0": [ 90, 1 ], "axis1": [ 80, 1 ], "diagonal": true, "matrix": "L28.mlp.W_out" }, { "axis0": [ 91, 4 ], "axis1": [ 217, 4 ], "diagonal": true, "matrix": "L28.mlp.W_out" }, { "axis0": [ 95, 1 ], "axis1": [ 222, 1 ], "diagonal": true, "matrix": "L28.mlp.W_out" }, { "axis0": [ 96, 7 ], "axis1": [ 274, 7 ], "diagonal": true, "matrix": "L28.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 103, 34 ], "matrix": "L28.mlp.W_up" }, { "axis0": [ 57, 1 ], "axis1": [ 103, 17 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 77, 2 ], "axis1": [ 103, 17 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 281, 4 ], "axis1": [ 103, 17 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 288, 2 ], "axis1": [ 103, 17 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 302, 1 ], "axis1": [ 103, 17 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 337, 1 ], "axis1": [ 103, 17 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 339, 1 ], "axis1": [ 103, 17 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 1909, 5 ], "axis1": [ 103, 17 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 57, 1 ], "axis1": [ 120, 17 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 77, 2 ], "axis1": [ 120, 17 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 281, 4 ], "axis1": [ 120, 17 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 288, 2 ], "axis1": [ 120, 17 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 302, 1 ], "axis1": [ 120, 17 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 337, 1 ], "axis1": [ 120, 17 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 339, 1 ], "axis1": [ 120, 17 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 1909, 5 ], "axis1": [ 120, 17 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 103, 1 ], "axis1": [ 57, 1 ], "diagonal": true, "matrix": "L28.mlp.W_out" }, { "axis0": [ 104, 2 ], "axis1": [ 77, 2 ], "diagonal": true, "matrix": "L28.mlp.W_out" }, { "axis0": [ 106, 4 ], "axis1": [ 281, 4 ], "diagonal": true, "matrix": "L28.mlp.W_out" }, { "axis0": [ 110, 2 ], "axis1": [ 288, 2 ], "diagonal": true, "matrix": "L28.mlp.W_out" }, { "axis0": [ 112, 1 ], "axis1": [ 302, 1 ], "diagonal": true, "matrix": "L28.mlp.W_out" }, { "axis0": [ 113, 1 ], "axis1": [ 337, 1 ], "diagonal": true, "matrix": "L28.mlp.W_out" }, { "axis0": [ 114, 1 ], "axis1": [ 339, 1 ], "diagonal": true, "matrix": "L28.mlp.W_out" }, { "axis0": [ 115, 5 ], "axis1": [ 1909, 5 ], "diagonal": true, "matrix": "L28.mlp.W_out" }, { "axis0": [ 120, 1 ], "axis1": [ 57, 1 ], "diagonal": true, "matrix": "L28.mlp.W_out" }, { "axis0": [ 121, 2 ], "axis1": [ 77, 2 ], "diagonal": true, "matrix": "L28.mlp.W_out" }, { "axis0": [ 123, 4 ], "axis1": [ 281, 4 ], "diagonal": true, "matrix": "L28.mlp.W_out" }, { "axis0": [ 127, 2 ], "axis1": [ 288, 2 ], "diagonal": true, "matrix": "L28.mlp.W_out" }, { "axis0": [ 129, 1 ], "axis1": [ 302, 1 ], "diagonal": true, "matrix": "L28.mlp.W_out" }, { "axis0": [ 130, 1 ], "axis1": [ 337, 1 ], "diagonal": true, "matrix": "L28.mlp.W_out" }, { "axis0": [ 131, 1 ], "axis1": [ 339, 1 ], "diagonal": true, "matrix": "L28.mlp.W_out" }, { "axis0": [ 132, 5 ], "axis1": [ 1909, 5 ], "diagonal": true, "matrix": "L28.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 137, 34 ], "matrix": "L28.mlp.W_up" }, { "axis0": [ 1914, 17 ], "axis1": [ 137, 17 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 1914, 17 ], "axis1": [ 154, 17 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 137, 17 ], "axis1": [ 1914, 17 ], "diagonal": true, "matrix": "L28.mlp.W_out" }, { "axis0": [ 154, 17 ], "axis1": [ 1914, 17 ], "diagonal": true, "matrix": "L28.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 171, 34 ], "matrix": "L28.mlp.W_up" }, { "axis0": [ 1948, 17 ], "axis1": [ 171, 17 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 1948, 17 ], "axis1": [ 188, 17 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 171, 17 ], "axis1": [ 1948, 17 ], "diagonal": true, "matrix": "L28.mlp.W_out" }, { "axis0": [ 188, 17 ], "axis1": [ 1948, 17 ], "diagonal": true, "matrix": "L28.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 205, 34 ], "matrix": "L28.mlp.W_up" }, { "axis0": [ 1965, 17 ], "axis1": [ 205, 17 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 1965, 17 ], "axis1": [ 222, 17 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 205, 17 ], "axis1": [ 1965, 17 ], "diagonal": true, "matrix": "L28.mlp.W_out" }, { "axis0": [ 222, 17 ], "axis1": [ 1965, 17 ], "diagonal": true, "matrix": "L28.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 239, 34 ], "matrix": "L28.mlp.W_up" }, { "axis0": [ 1982, 17 ], "axis1": [ 239, 17 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 1982, 17 ], "axis1": [ 256, 17 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 239, 17 ], "axis1": [ 1982, 17 ], "diagonal": true, "matrix": "L28.mlp.W_out" }, { "axis0": [ 256, 17 ], "axis1": [ 1982, 17 ], "diagonal": true, "matrix": "L28.mlp.W_out" }, { "axis0": [ 0, 1 ], "axis1": [ 273, 34 ], "matrix": "L28.mlp.W_up" }, { "axis0": [ 1999, 17 ], "axis1": [ 273, 17 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 1999, 17 ], "axis1": [ 290, 17 ], "matrix": "L28.mlp.W_in" }, { "axis0": [ 273, 17 ], "axis1": [ 1999, 17 ], "diagonal": true, "matrix": "L28.mlp.W_out" }, { "axis0": [ 290, 17 ], "axis1": [ 1999, 17 ], "diagonal": true, "matrix": "L28.mlp.W_out" } ] } }, "reference_fixtures": { "included": [], "reason": "No input corpus was supplied to compile_hf_bundle.", "scope": "Input-specific activations and intervention results." }, "residual_stream": { "constant_one_column": 0, "constant_one_semantics": "Holds exactly 1.0 at every position. Read as the \u0394=0 self-match query/key source by every attention operation except compute_attn (their query/key source columns) and, under bias=False, by MLP bias folds (their W_in/W_up rows in physical_layout.placements). Patching it collapses every self-match head.", "rms_norm_reserved_columns": [ 2046, 2047 ], "state_semantics": "input and post-layer residual streams; the compiler has no stable observation boundary between attention and MLP", "states": [ { "key": "input", "nodes": { "i:0": [ [ 0, 1 ] ], "l:8": [ [ 1, 17 ] ] } }, { "key": "L0.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:100": [ [ 33, 1 ] ], "l:103": [ [ 27, 1 ] ], "l:104": [ [ 28, 1 ] ], "l:106": [ [ 22, 1 ] ], "l:107": [ [ 29, 1 ] ], "l:115": [ [ 23, 1 ] ], "l:131": [ [ 35, 4 ] ], "l:152": [ [ 34, 1 ] ], "l:155": [ [ 30, 1 ] ], "l:156": [ [ 31, 1 ] ], "l:158": [ [ 32, 1 ] ], "l:4": [ [ 39, 1 ] ], "l:46": [ [ 18, 1 ] ], "l:48": [ [ 19, 1 ] ], "l:5": [ [ 40, 1 ] ], "l:50": [ [ 20, 1 ] ], "l:54": [ [ 24, 1 ] ], "l:55": [ [ 25, 1 ] ], "l:57": [ [ 26, 1 ] ], "l:7": [ [ 21, 1 ] ], "l:8": [ [ 1, 17 ] ], "l:9": [ [ 41, 1 ] ] } }, { "key": "L1.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:100": [ [ 33, 1 ] ], "l:101": [ [ 25, 1 ] ], "l:108": [ [ 46, 1 ] ], "l:115": [ [ 23, 1 ] ], "l:129": [ [ 42, 4 ] ], "l:152": [ [ 34, 1 ] ], "l:155": [ [ 30, 1 ] ], "l:156": [ [ 31, 1 ] ], "l:158": [ [ 32, 1 ] ], "l:166": [ [ 39, 1 ] ], "l:2": [ [ 26, 1 ] ], "l:20": [ [ 48, 1 ] ], "l:21": [ [ 47, 1 ] ], "l:45": [ [ 27, 1 ] ], "l:46": [ [ 18, 1 ] ], "l:47": [ [ 28, 1 ] ], "l:48": [ [ 19, 1 ] ], "l:49": [ [ 29, 1 ] ], "l:50": [ [ 20, 1 ] ], "l:51": [ [ 40, 1 ] ], "l:52": [ [ 24, 1 ] ], "l:7": [ [ 21, 1 ] ], "l:8": [ [ 1, 17 ] ] } }, { "key": "L2.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:115": [ [ 23, 1 ] ], "l:129": [ [ 42, 4 ] ], "l:152": [ [ 34, 1 ] ], "l:155": [ [ 30, 1 ] ], "l:156": [ [ 31, 1 ] ], "l:158": [ [ 32, 1 ] ], "l:166": [ [ 39, 1 ] ], "l:18": [ [ 35, 1 ] ], "l:19": [ [ 24, 1 ] ], "l:2": [ [ 26, 1 ] ], "l:43": [ [ 25, 1 ] ], "l:46": [ [ 18, 1 ] ], "l:48": [ [ 19, 1 ] ], "l:50": [ [ 20, 1 ] ], "l:51": [ [ 40, 1 ] ], "l:7": [ [ 21, 1 ] ], "l:8": [ [ 1, 17 ] ], "l:98": [ [ 33, 1 ] ], "l:99": [ [ 22, 1 ] ] } }, { "key": "L3.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:115": [ [ 23, 1 ] ], "l:119": [ [ 50, 1 ] ], "l:120": [ [ 51, 1 ] ], "l:122": [ [ 52, 1 ] ], "l:126": [ [ 49, 1 ] ], "l:129": [ [ 42, 4 ] ], "l:152": [ [ 34, 1 ] ], "l:155": [ [ 30, 1 ] ], "l:156": [ [ 31, 1 ] ], "l:158": [ [ 32, 1 ] ], "l:16": [ [ 28, 1 ] ], "l:166": [ [ 39, 1 ] ], "l:17": [ [ 24, 1 ] ], "l:175": [ [ 53, 4 ] ], "l:2": [ [ 26, 1 ] ], "l:22": [ [ 29, 1 ] ], "l:38": [ [ 60, 1 ] ], "l:39": [ [ 61, 1 ] ], "l:41": [ [ 27, 1 ] ], "l:428": [ [ 36, 1 ] ], "l:429": [ [ 37, 1 ] ], "l:438": [ [ 86, 17 ] ], "l:441": [ [ 103, 17 ] ], "l:444": [ [ 120, 17 ] ], "l:46": [ [ 18, 1 ] ], "l:48": [ [ 19, 1 ] ], "l:487": [ [ 38, 1 ] ], "l:488": [ [ 46, 1 ] ], "l:50": [ [ 20, 1 ] ], "l:58": [ [ 62, 1 ] ], "l:59": [ [ 69, 17 ] ], "l:64": [ [ 63, 1 ] ], "l:65": [ [ 64, 1 ] ], "l:67": [ [ 65, 1 ] ], "l:7": [ [ 21, 1 ] ], "l:73": [ [ 66, 1 ] ], "l:74": [ [ 67, 1 ] ], "l:76": [ [ 68, 1 ] ], "l:8": [ [ 1, 17 ] ], "l:93": [ [ 57, 1 ] ], "l:94": [ [ 58, 1 ] ], "l:96": [ [ 59, 1 ] ], "l:97": [ [ 22, 1 ] ] } }, { "key": "L4.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:115": [ [ 23, 1 ] ], "l:116": [ [ 51, 1 ] ], "l:117": [ [ 31, 1 ] ], "l:124": [ [ 52, 1 ], [ 57, 2 ] ], "l:127": [ [ 50, 1 ] ], "l:129": [ [ 42, 4 ] ], "l:14": [ [ 285, 1 ] ], "l:152": [ [ 34, 1 ] ], "l:153": [ [ 30, 1 ] ], "l:159": [ [ 288, 1 ] ], "l:16": [ [ 28, 1 ] ], "l:164": [ [ 59, 1 ] ], "l:173": [ [ 60, 4 ] ], "l:178": [ [ 289, 17 ] ], "l:2": [ [ 26, 1 ] ], "l:201": [ [ 18, 3 ], [ 68, 1 ] ], "l:22": [ [ 29, 1 ] ], "l:221": [ [ 64, 4 ] ], "l:277": [ [ 156, 17 ] ], "l:34": [ [ 139, 17 ] ], "l:36": [ [ 33, 1 ] ], "l:426": [ [ 286, 1 ] ], "l:428": [ [ 36, 1 ] ], "l:429": [ [ 37, 1 ] ], "l:436": [ [ 173, 17 ] ], "l:439": [ [ 190, 17 ] ], "l:442": [ [ 207, 17 ] ], "l:449": [ [ 275, 1 ] ], "l:450": [ [ 276, 1 ] ], "l:457": [ [ 224, 17 ] ], "l:459": [ [ 241, 17 ] ], "l:461": [ [ 258, 17 ] ], "l:467": [ [ 277, 1 ] ], "l:468": [ [ 278, 1 ] ], "l:477": [ [ 279, 1 ] ], "l:478": [ [ 280, 1 ] ], "l:485": [ [ 287, 1 ] ], "l:487": [ [ 38, 1 ] ], "l:488": [ [ 46, 1 ] ], "l:497": [ [ 281, 1 ] ], "l:498": [ [ 282, 1 ] ], "l:507": [ [ 283, 1 ] ], "l:508": [ [ 284, 1 ] ], "l:62": [ [ 35, 1 ] ], "l:68": [ [ 306, 17 ] ], "l:7": [ [ 21, 1 ] ], "l:71": [ [ 40, 1 ] ], "l:8": [ [ 1, 17 ] ], "l:90": [ [ 22, 1 ], [ 25, 1 ], [ 137, 2 ] ], "l:91": [ [ 32, 1 ] ] } }, { "key": "L5.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:113": [ [ 103, 1 ] ], "l:123": [ [ 236, 3 ] ], "l:129": [ [ 42, 4 ] ], "l:134": [ [ 49, 2 ], [ 53, 4 ], [ 239, 2 ], [ 288, 1 ], [ 323, 8 ] ], "l:14": [ [ 285, 1 ] ], "l:150": [ [ 102, 1 ] ], "l:151": [ [ 24, 1 ] ], "l:164": [ [ 59, 1 ] ], "l:173": [ [ 60, 4 ] ], "l:176": [ [ 104, 17 ] ], "l:2": [ [ 26, 1 ] ], "l:201": [ [ 18, 3 ], [ 68, 1 ] ], "l:221": [ [ 64, 4 ] ], "l:277": [ [ 156, 17 ] ], "l:279": [ [ 29, 4 ], [ 34, 1 ], [ 36, 4 ], [ 190, 8 ] ], "l:314": [ [ 46, 1 ], [ 198, 9 ], [ 224, 7 ] ], "l:34": [ [ 139, 17 ] ], "l:36": [ [ 33, 1 ] ], "l:426": [ [ 286, 1 ] ], "l:434": [ [ 27, 1 ], [ 69, 16 ] ], "l:447": [ [ 231, 1 ] ], "l:455": [ [ 85, 17 ] ], "l:465": [ [ 232, 1 ] ], "l:468": [ [ 278, 1 ] ], "l:475": [ [ 233, 1 ] ], "l:478": [ [ 280, 1 ] ], "l:485": [ [ 287, 1 ] ], "l:495": [ [ 234, 1 ] ], "l:497": [ [ 281, 1 ] ], "l:505": [ [ 235, 1 ] ], "l:60": [ [ 28, 1 ], [ 121, 16 ] ], "l:62": [ [ 35, 1 ] ], "l:7": [ [ 21, 1 ] ], "l:71": [ [ 40, 1 ] ], "l:8": [ [ 1, 17 ] ], "l:90": [ [ 22, 1 ], [ 25, 1 ], [ 137, 2 ] ] } }, { "key": "L6.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:111": [ [ 23, 1 ], [ 52, 1 ], [ 102, 1 ] ], "l:128": [ [ 210, 1 ] ], "l:129": [ [ 42, 4 ] ], "l:132": [ [ 24, 1 ], [ 173, 16 ] ], "l:14": [ [ 285, 1 ] ], "l:145": [ [ 207, 1 ] ], "l:146": [ [ 208, 1 ] ], "l:148": [ [ 209, 1 ] ], "l:149": [ [ 51, 1 ] ], "l:164": [ [ 59, 1 ] ], "l:170": [ [ 189, 1 ] ], "l:173": [ [ 60, 4 ] ], "l:176": [ [ 104, 17 ] ], "l:2": [ [ 26, 1 ] ], "l:201": [ [ 18, 3 ], [ 68, 1 ] ], "l:221": [ [ 64, 4 ] ], "l:277": [ [ 156, 17 ] ], "l:279": [ [ 29, 4 ], [ 34, 1 ], [ 36, 4 ], [ 190, 8 ] ], "l:314": [ [ 46, 1 ], [ 198, 9 ], [ 224, 7 ] ], "l:34": [ [ 139, 17 ] ], "l:36": [ [ 33, 1 ] ], "l:426": [ [ 286, 1 ] ], "l:434": [ [ 27, 1 ], [ 69, 16 ] ], "l:447": [ [ 231, 1 ] ], "l:455": [ [ 85, 17 ] ], "l:465": [ [ 232, 1 ] ], "l:475": [ [ 233, 1 ] ], "l:485": [ [ 287, 1 ] ], "l:495": [ [ 234, 1 ] ], "l:505": [ [ 235, 1 ] ], "l:60": [ [ 28, 1 ], [ 121, 16 ] ], "l:62": [ [ 35, 1 ] ], "l:7": [ [ 21, 1 ] ], "l:71": [ [ 40, 1 ] ], "l:8": [ [ 1, 17 ] ], "l:90": [ [ 22, 1 ], [ 25, 1 ], [ 137, 2 ] ] } }, { "key": "L7.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:109": [ [ 42, 4 ] ], "l:111": [ [ 23, 1 ], [ 52, 1 ], [ 102, 1 ] ], "l:128": [ [ 210, 1 ] ], "l:132": [ [ 24, 1 ], [ 173, 16 ] ], "l:14": [ [ 285, 1 ] ], "l:142": [ [ 208, 2 ], [ 211, 2 ] ], "l:143": [ [ 326, 1 ] ], "l:164": [ [ 59, 1 ] ], "l:168": [ [ 51, 1 ], [ 53, 2 ] ], "l:171": [ [ 50, 1 ] ], "l:173": [ [ 60, 4 ] ], "l:176": [ [ 104, 17 ] ], "l:191": [ [ 103, 1 ], [ 207, 1 ], [ 327, 1 ], [ 330, 1 ] ], "l:2": [ [ 26, 1 ] ], "l:201": [ [ 18, 3 ], [ 68, 1 ] ], "l:221": [ [ 64, 4 ] ], "l:229": [ [ 55, 2 ], [ 328, 2 ] ], "l:277": [ [ 156, 17 ] ], "l:279": [ [ 29, 4 ], [ 34, 1 ], [ 36, 4 ], [ 190, 8 ] ], "l:314": [ [ 46, 1 ], [ 198, 9 ], [ 224, 7 ] ], "l:34": [ [ 139, 17 ] ], "l:36": [ [ 33, 1 ] ], "l:426": [ [ 286, 1 ] ], "l:434": [ [ 27, 1 ], [ 69, 16 ] ], "l:447": [ [ 231, 1 ] ], "l:455": [ [ 85, 17 ] ], "l:465": [ [ 232, 1 ] ], "l:475": [ [ 233, 1 ] ], "l:485": [ [ 287, 1 ] ], "l:495": [ [ 234, 1 ] ], "l:505": [ [ 235, 1 ] ], "l:60": [ [ 28, 1 ], [ 121, 16 ] ], "l:62": [ [ 35, 1 ] ], "l:7": [ [ 21, 1 ] ], "l:71": [ [ 40, 1 ] ], "l:90": [ [ 22, 1 ], [ 25, 1 ], [ 137, 2 ] ] } }, { "key": "L8.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:14": [ [ 285, 1 ] ], "l:142": [ [ 208, 2 ], [ 211, 2 ] ], "l:162": [ [ 236, 3 ] ], "l:167": [ [ 45, 1 ], [ 222, 2 ] ], "l:172": [ [ 49, 1 ] ], "l:173": [ [ 60, 4 ] ], "l:176": [ [ 104, 17 ] ], "l:191": [ [ 103, 1 ], [ 207, 1 ], [ 327, 1 ], [ 330, 1 ] ], "l:2": [ [ 26, 1 ] ], "l:200": [ [ 42, 2 ], [ 185, 5 ], [ 210, 1 ], [ 213, 9 ] ], "l:220": [ [ 18, 3 ], [ 22, 2 ], [ 44, 1 ], [ 50, 5 ], [ 64, 2 ], [ 67, 2 ], [ 102, 1 ], [ 326, 1 ] ], "l:229": [ [ 55, 2 ], [ 328, 2 ] ], "l:277": [ [ 156, 17 ] ], "l:279": [ [ 29, 4 ], [ 34, 1 ], [ 36, 4 ], [ 190, 8 ] ], "l:314": [ [ 46, 1 ], [ 198, 9 ], [ 224, 7 ] ], "l:34": [ [ 139, 17 ] ], "l:36": [ [ 33, 1 ] ], "l:426": [ [ 286, 1 ] ], "l:434": [ [ 27, 1 ], [ 69, 16 ] ], "l:447": [ [ 231, 1 ] ], "l:455": [ [ 85, 17 ] ], "l:465": [ [ 232, 1 ] ], "l:475": [ [ 233, 1 ] ], "l:485": [ [ 287, 1 ] ], "l:495": [ [ 234, 1 ] ], "l:505": [ [ 235, 1 ] ], "l:60": [ [ 28, 1 ], [ 121, 16 ] ], "l:62": [ [ 35, 1 ] ], "l:7": [ [ 21, 1 ] ], "l:71": [ [ 40, 1 ] ], "l:86": [ [ 239, 1 ] ], "l:89": [ [ 24, 2 ], [ 66, 1 ], [ 137, 2 ], [ 173, 12 ] ] } }, { "key": "L9.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:14": [ [ 285, 1 ] ], "l:142": [ [ 208, 2 ], [ 211, 2 ] ], "l:160": [ [ 60, 4 ] ], "l:162": [ [ 236, 3 ] ], "l:172": [ [ 49, 1 ] ], "l:176": [ [ 104, 17 ] ], "l:191": [ [ 103, 1 ], [ 207, 1 ], [ 327, 1 ], [ 330, 1 ] ], "l:2": [ [ 26, 1 ] ], "l:200": [ [ 42, 2 ], [ 185, 5 ], [ 210, 1 ], [ 213, 9 ] ], "l:220": [ [ 18, 3 ], [ 22, 2 ], [ 44, 1 ], [ 50, 5 ], [ 64, 2 ], [ 67, 2 ], [ 102, 1 ], [ 326, 1 ] ], "l:229": [ [ 55, 2 ], [ 328, 2 ] ], "l:277": [ [ 156, 17 ] ], "l:279": [ [ 29, 4 ], [ 34, 1 ], [ 36, 4 ], [ 190, 8 ] ], "l:314": [ [ 46, 1 ], [ 198, 9 ], [ 224, 7 ] ], "l:34": [ [ 139, 17 ] ], "l:36": [ [ 33, 1 ] ], "l:426": [ [ 286, 1 ] ], "l:434": [ [ 27, 1 ], [ 69, 16 ] ], "l:447": [ [ 231, 1 ] ], "l:455": [ [ 85, 17 ] ], "l:465": [ [ 232, 1 ] ], "l:475": [ [ 233, 1 ] ], "l:485": [ [ 287, 1 ] ], "l:495": [ [ 234, 1 ] ], "l:505": [ [ 235, 1 ] ], "l:60": [ [ 28, 1 ], [ 121, 16 ] ], "l:62": [ [ 35, 1 ] ], "l:7": [ [ 21, 1 ] ], "l:71": [ [ 40, 1 ] ], "l:86": [ [ 239, 1 ] ], "l:89": [ [ 24, 2 ], [ 66, 1 ], [ 137, 2 ], [ 173, 12 ] ] } }, { "key": "L10.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:137": [ [ 263, 1 ] ], "l:138": [ [ 264, 1 ] ], "l:14": [ [ 285, 1 ] ], "l:140": [ [ 265, 1 ] ], "l:141": [ [ 111, 10 ], [ 207, 3 ], [ 211, 2 ], [ 222, 1 ], [ 327, 1 ] ], "l:186": [ [ 266, 1 ] ], "l:187": [ [ 267, 1 ] ], "l:189": [ [ 268, 1 ] ], "l:190": [ [ 45, 1 ], [ 223, 1 ], [ 236, 3 ], [ 240, 12 ] ], "l:196": [ [ 260, 1 ] ], "l:197": [ [ 261, 1 ] ], "l:199": [ [ 262, 1 ] ], "l:2": [ [ 26, 1 ] ], "l:200": [ [ 42, 2 ], [ 185, 5 ], [ 210, 1 ], [ 213, 9 ] ], "l:216": [ [ 252, 1 ] ], "l:217": [ [ 253, 1 ] ], "l:219": [ [ 254, 1 ] ], "l:220": [ [ 18, 3 ], [ 22, 2 ], [ 44, 1 ], [ 50, 5 ], [ 64, 2 ], [ 67, 2 ], [ 102, 1 ], [ 326, 1 ] ], "l:224": [ [ 255, 1 ] ], "l:225": [ [ 256, 1 ] ], "l:227": [ [ 257, 1 ] ], "l:228": [ [ 55, 2 ], [ 59, 5 ], [ 103, 8 ], [ 328, 2 ] ], "l:277": [ [ 156, 17 ] ], "l:279": [ [ 29, 4 ], [ 34, 1 ], [ 36, 4 ], [ 190, 8 ] ], "l:314": [ [ 46, 1 ], [ 198, 9 ], [ 224, 7 ] ], "l:34": [ [ 139, 17 ] ], "l:36": [ [ 33, 1 ] ], "l:426": [ [ 286, 1 ] ], "l:434": [ [ 27, 1 ], [ 69, 16 ] ], "l:447": [ [ 231, 1 ] ], "l:455": [ [ 85, 17 ] ], "l:465": [ [ 232, 1 ] ], "l:475": [ [ 233, 1 ] ], "l:485": [ [ 287, 1 ] ], "l:495": [ [ 234, 1 ] ], "l:505": [ [ 235, 1 ] ], "l:60": [ [ 28, 1 ], [ 121, 16 ] ], "l:62": [ [ 35, 1 ] ], "l:7": [ [ 21, 1 ] ], "l:71": [ [ 40, 1 ] ], "l:85": [ [ 258, 1 ] ], "l:86": [ [ 239, 1 ] ], "l:88": [ [ 259, 1 ] ], "l:89": [ [ 24, 2 ], [ 66, 1 ], [ 137, 2 ], [ 173, 12 ] ] } }, { "key": "L11.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:135": [ [ 49, 2 ], [ 239, 8 ], [ 248, 3 ], [ 252, 4 ] ], "l:14": [ [ 285, 1 ] ], "l:182": [ [ 351, 34 ] ], "l:184": [ [ 51, 4 ], [ 67, 1 ], [ 258, 5 ], [ 266, 7 ] ], "l:192": [ [ 419, 34 ] ], "l:194": [ [ 42, 4 ], [ 213, 9 ], [ 223, 1 ], [ 236, 3 ] ], "l:2": [ [ 26, 1 ] ], "l:212": [ [ 57, 2 ], [ 273, 12 ], [ 288, 20 ] ], "l:214": [ [ 18, 3 ], [ 55, 2 ], [ 59, 7 ], [ 68, 1 ], [ 106, 1 ], [ 326, 1 ], [ 328, 2 ] ], "l:222": [ [ 21, 5 ], [ 66, 1 ], [ 102, 4 ], [ 107, 4 ], [ 137, 2 ], [ 173, 1 ] ], "l:241": [ [ 487, 34 ] ], "l:243": [ [ 555, 34 ] ], "l:251": [ [ 385, 34 ] ], "l:253": [ [ 453, 34 ] ], "l:255": [ [ 521, 34 ] ], "l:277": [ [ 156, 17 ] ], "l:279": [ [ 29, 4 ], [ 34, 1 ], [ 36, 4 ], [ 190, 8 ] ], "l:306": [ [ 598, 3 ] ], "l:310": [ [ 592, 3 ] ], "l:312": [ [ 595, 3 ] ], "l:314": [ [ 46, 1 ], [ 198, 9 ], [ 224, 7 ] ], "l:34": [ [ 139, 17 ] ], "l:347": [ [ 589, 1 ] ], "l:351": [ [ 590, 1 ] ], "l:355": [ [ 591, 1 ] ], "l:36": [ [ 33, 1 ] ], "l:426": [ [ 286, 1 ] ], "l:434": [ [ 27, 1 ], [ 69, 16 ] ], "l:447": [ [ 231, 1 ] ], "l:455": [ [ 85, 17 ] ], "l:465": [ [ 232, 1 ] ], "l:475": [ [ 233, 1 ] ], "l:485": [ [ 287, 1 ] ], "l:495": [ [ 234, 1 ] ], "l:505": [ [ 235, 1 ] ], "l:60": [ [ 28, 1 ], [ 121, 16 ] ], "l:62": [ [ 35, 1 ] ], "l:71": [ [ 40, 1 ] ], "l:81": [ [ 308, 11 ], [ 320, 3 ], [ 331, 20 ] ], "l:83": [ [ 174, 16 ], [ 210, 1 ] ] } }, { "key": "L12.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:135": [ [ 49, 2 ], [ 239, 8 ], [ 248, 3 ], [ 252, 4 ] ], "l:14": [ [ 285, 1 ] ], "l:180": [ [ 208, 1 ] ], "l:184": [ [ 51, 4 ], [ 67, 1 ], [ 258, 5 ], [ 266, 7 ] ], "l:194": [ [ 42, 4 ], [ 213, 9 ], [ 223, 1 ], [ 236, 3 ] ], "l:2": [ [ 26, 1 ] ], "l:211": [ [ 120, 1 ] ], "l:214": [ [ 18, 3 ], [ 55, 2 ], [ 59, 7 ], [ 68, 1 ], [ 106, 1 ], [ 326, 1 ], [ 328, 2 ] ], "l:222": [ [ 21, 5 ], [ 66, 1 ], [ 102, 4 ], [ 107, 4 ], [ 137, 2 ], [ 173, 1 ] ], "l:230": [ [ 355, 1 ] ], "l:239": [ [ 356, 1 ] ], "l:249": [ [ 327, 1 ] ], "l:263": [ [ 207, 1 ] ], "l:277": [ [ 156, 17 ] ], "l:279": [ [ 29, 4 ], [ 34, 1 ], [ 36, 4 ], [ 190, 8 ] ], "l:306": [ [ 598, 3 ] ], "l:308": [ [ 264, 2 ], [ 351, 1 ] ], "l:314": [ [ 46, 1 ], [ 198, 9 ], [ 224, 7 ] ], "l:335": [ [ 352, 3 ] ], "l:34": [ [ 139, 17 ] ], "l:346": [ [ 209, 1 ] ], "l:349": [ [ 211, 1 ] ], "l:350": [ [ 212, 1 ] ], "l:353": [ [ 222, 1 ] ], "l:354": [ [ 256, 1 ] ], "l:357": [ [ 263, 1 ] ], "l:36": [ [ 33, 1 ] ], "l:390": [ [ 358, 2 ] ], "l:412": [ [ 360, 2 ] ], "l:426": [ [ 286, 1 ] ], "l:434": [ [ 27, 1 ], [ 69, 16 ] ], "l:447": [ [ 231, 1 ] ], "l:455": [ [ 85, 17 ] ], "l:465": [ [ 232, 1 ] ], "l:475": [ [ 233, 1 ] ], "l:485": [ [ 287, 1 ] ], "l:495": [ [ 234, 1 ] ], "l:505": [ [ 235, 1 ] ], "l:60": [ [ 28, 1 ], [ 121, 16 ] ], "l:62": [ [ 35, 1 ] ], "l:71": [ [ 40, 1 ] ], "l:80": [ [ 357, 1 ] ], "l:83": [ [ 174, 16 ], [ 210, 1 ] ] } }, { "key": "L13.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:135": [ [ 49, 2 ], [ 239, 8 ], [ 248, 3 ], [ 252, 4 ] ], "l:14": [ [ 285, 1 ] ], "l:180": [ [ 208, 1 ] ], "l:184": [ [ 51, 4 ], [ 67, 1 ], [ 258, 5 ], [ 266, 7 ] ], "l:194": [ [ 42, 4 ], [ 213, 9 ], [ 223, 1 ], [ 236, 3 ] ], "l:2": [ [ 26, 1 ] ], "l:209": [ [ 57, 1 ], [ 211, 1 ], [ 222, 1 ], [ 263, 1 ], [ 273, 12 ], [ 288, 16 ], [ 305, 2 ], [ 308, 11 ], [ 320, 3 ], [ 331, 11 ], [ 343, 2 ], [ 347, 1 ], [ 350, 1 ], [ 362, 27 ], [ 390, 9 ], [ 404, 1 ] ], "l:214": [ [ 18, 3 ], [ 55, 2 ], [ 59, 7 ], [ 68, 1 ], [ 106, 1 ], [ 326, 1 ], [ 328, 2 ] ], "l:222": [ [ 21, 5 ], [ 66, 1 ], [ 102, 4 ], [ 107, 4 ], [ 137, 2 ], [ 173, 1 ] ], "l:239": [ [ 356, 1 ] ], "l:248": [ [ 342, 1 ], [ 399, 5 ], [ 405, 9 ], [ 415, 2 ] ], "l:262": [ [ 434, 17 ] ], "l:265": [ [ 417, 17 ] ], "l:269": [ [ 451, 17 ] ], "l:277": [ [ 156, 17 ] ], "l:279": [ [ 29, 4 ], [ 34, 1 ], [ 36, 4 ], [ 190, 8 ] ], "l:302": [ [ 525, 17 ] ], "l:304": [ [ 469, 3 ] ], "l:314": [ [ 46, 1 ], [ 198, 9 ], [ 224, 7 ] ], "l:334": [ [ 523, 2 ] ], "l:34": [ [ 139, 17 ] ], "l:343": [ [ 468, 1 ] ], "l:344": [ [ 58, 1 ] ], "l:36": [ [ 33, 1 ] ], "l:388": [ [ 345, 1 ], [ 490, 16 ] ], "l:410": [ [ 346, 1 ], [ 506, 5 ], [ 512, 11 ] ], "l:412": [ [ 360, 2 ] ], "l:426": [ [ 286, 1 ] ], "l:434": [ [ 27, 1 ], [ 69, 16 ] ], "l:447": [ [ 231, 1 ] ], "l:455": [ [ 85, 17 ] ], "l:465": [ [ 232, 1 ] ], "l:475": [ [ 233, 1 ] ], "l:485": [ [ 287, 1 ] ], "l:495": [ [ 234, 1 ] ], "l:505": [ [ 235, 1 ] ], "l:60": [ [ 28, 1 ], [ 121, 16 ] ], "l:62": [ [ 35, 1 ] ], "l:71": [ [ 40, 1 ] ], "l:79": [ [ 472, 17 ] ], "l:83": [ [ 174, 16 ], [ 210, 1 ] ] } }, { "key": "L14.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:14": [ [ 285, 1 ] ], "l:180": [ [ 208, 1 ] ], "l:2": [ [ 26, 1 ] ], "l:208": [ [ 743, 100 ] ], "l:231": [ [ 843, 17 ] ], "l:238": [ [ 207, 1 ], [ 209, 1 ], [ 212, 1 ], [ 256, 1 ], [ 264, 2 ], [ 351, 5 ], [ 357, 5 ], [ 542, 1 ] ], "l:239": [ [ 356, 1 ] ], "l:248": [ [ 342, 1 ], [ 399, 5 ], [ 405, 9 ], [ 415, 2 ] ], "l:257": [ [ 111, 1 ], [ 543, 16 ] ], "l:260": [ [ 115, 1 ], [ 577, 2 ] ], "l:262": [ [ 434, 17 ] ], "l:265": [ [ 417, 17 ] ], "l:268": [ [ 112, 3 ], [ 348, 1 ], [ 560, 3 ], [ 565, 4 ], [ 570, 4 ], [ 575, 2 ] ], "l:269": [ [ 451, 17 ] ], "l:277": [ [ 156, 17 ] ], "l:279": [ [ 29, 4 ], [ 34, 1 ], [ 36, 4 ], [ 190, 8 ] ], "l:300": [ [ 708, 17 ] ], "l:303": [ [ 120, 1 ], [ 327, 1 ] ], "l:314": [ [ 46, 1 ], [ 198, 9 ], [ 224, 7 ] ], "l:332": [ [ 691, 17 ] ], "l:34": [ [ 139, 17 ] ], "l:341": [ [ 657, 17 ] ], "l:343": [ [ 468, 1 ] ], "l:358": [ [ 674, 17 ] ], "l:36": [ [ 33, 1 ] ], "l:365": [ [ 606, 17 ] ], "l:367": [ [ 640, 17 ] ], "l:371": [ [ 116, 4 ], [ 349, 1 ], [ 580, 4 ], [ 585, 3 ], [ 601, 5 ] ], "l:373": [ [ 623, 17 ] ], "l:388": [ [ 345, 1 ], [ 490, 16 ] ], "l:404": [ [ 725, 1 ] ], "l:408": [ [ 726, 17 ] ], "l:417": [ [ 860, 2 ] ], "l:422": [ [ 862, 2 ] ], "l:426": [ [ 286, 1 ] ], "l:434": [ [ 27, 1 ], [ 69, 16 ] ], "l:447": [ [ 231, 1 ] ], "l:455": [ [ 85, 17 ] ], "l:465": [ [ 232, 1 ] ], "l:475": [ [ 233, 1 ] ], "l:485": [ [ 287, 1 ] ], "l:495": [ [ 234, 1 ] ], "l:505": [ [ 235, 1 ] ], "l:60": [ [ 28, 1 ], [ 121, 16 ] ], "l:62": [ [ 35, 1 ] ], "l:71": [ [ 40, 1 ] ], "l:79": [ [ 472, 17 ] ] } }, { "key": "L15.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:14": [ [ 285, 1 ] ], "l:180": [ [ 208, 1 ] ], "l:2": [ [ 26, 1 ] ], "l:205": [ [ 44, 1 ], [ 220, 1 ], [ 222, 1 ] ], "l:207": [ [ 177, 2 ], [ 180, 9 ], [ 210, 1 ], [ 214, 1 ], [ 217, 1 ], [ 333, 1 ], [ 392, 1 ], [ 404, 1 ] ], "l:236": [ [ 42, 1 ], [ 211, 1 ], [ 213, 1 ] ], "l:238": [ [ 207, 1 ], [ 209, 1 ], [ 212, 1 ], [ 256, 1 ], [ 264, 2 ], [ 351, 5 ], [ 357, 5 ], [ 542, 1 ] ], "l:245": [ [ 52, 3 ], [ 56, 1 ], [ 58, 6 ], [ 65, 1 ], [ 173, 4 ], [ 179, 1 ], [ 326, 1 ] ], "l:246": [ [ 215, 2 ], [ 334, 1 ] ], "l:248": [ [ 342, 1 ], [ 399, 5 ], [ 405, 9 ], [ 415, 2 ] ], "l:257": [ [ 111, 1 ], [ 543, 16 ] ], "l:260": [ [ 115, 1 ], [ 577, 2 ] ], "l:262": [ [ 434, 17 ] ], "l:265": [ [ 417, 17 ] ], "l:266": [ [ 43, 1 ], [ 218, 2 ] ], "l:268": [ [ 112, 3 ], [ 348, 1 ], [ 560, 3 ], [ 565, 4 ], [ 570, 4 ], [ 575, 2 ] ], "l:269": [ [ 451, 17 ] ], "l:277": [ [ 156, 17 ] ], "l:279": [ [ 29, 4 ], [ 34, 1 ], [ 36, 4 ], [ 190, 8 ] ], "l:298": [ [ 49, 2 ], [ 240, 7 ], [ 248, 3 ], [ 252, 4 ], [ 335, 1 ] ], "l:314": [ [ 46, 1 ], [ 198, 9 ], [ 224, 7 ] ], "l:330": [ [ 51, 1 ], [ 55, 1 ], [ 67, 1 ], [ 258, 6 ], [ 266, 6 ], [ 336, 1 ], [ 391, 1 ] ], "l:34": [ [ 139, 17 ] ], "l:341": [ [ 657, 17 ] ], "l:358": [ [ 674, 17 ] ], "l:363": [ [ 45, 1 ], [ 223, 1 ], [ 239, 1 ] ], "l:369": [ [ 236, 3 ] ], "l:371": [ [ 116, 4 ], [ 349, 1 ], [ 580, 4 ], [ 585, 3 ], [ 601, 5 ] ], "l:373": [ [ 623, 17 ] ], "l:386": [ [ 57, 1 ], [ 273, 12 ], [ 288, 2 ], [ 337, 1 ], [ 339, 1 ] ], "l:402": [ [ 68, 1 ] ], "l:408": [ [ 726, 17 ] ], "l:415": [ [ 290, 14 ], [ 305, 2 ], [ 393, 1 ] ], "l:420": [ [ 64, 1 ], [ 308, 11 ], [ 320, 3 ], [ 328, 1 ], [ 338, 1 ] ], "l:426": [ [ 286, 1 ] ], "l:434": [ [ 27, 1 ], [ 69, 16 ] ], "l:447": [ [ 231, 1 ] ], "l:455": [ [ 85, 17 ] ], "l:465": [ [ 232, 1 ] ], "l:475": [ [ 233, 1 ] ], "l:485": [ [ 287, 1 ] ], "l:495": [ [ 234, 1 ] ], "l:505": [ [ 235, 1 ] ], "l:60": [ [ 28, 1 ], [ 121, 16 ] ], "l:62": [ [ 35, 1 ] ], "l:71": [ [ 40, 1 ] ], "l:79": [ [ 472, 17 ] ] } }, { "key": "L16.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:14": [ [ 285, 1 ] ], "l:179": [ [ 341, 1 ], [ 343, 1 ], [ 377, 12 ], [ 390, 1 ], [ 394, 1 ], [ 398, 1 ] ], "l:2": [ [ 26, 1 ] ], "l:205": [ [ 44, 1 ], [ 220, 1 ], [ 222, 1 ] ], "l:234": [ [ 340, 1 ], [ 362, 2 ] ], "l:248": [ [ 342, 1 ], [ 399, 5 ], [ 405, 9 ], [ 415, 2 ] ], "l:257": [ [ 111, 1 ], [ 543, 16 ] ], "l:258": [ [ 365, 3 ] ], "l:262": [ [ 434, 17 ] ], "l:265": [ [ 417, 17 ] ], "l:266": [ [ 43, 1 ], [ 218, 2 ] ], "l:268": [ [ 112, 3 ], [ 348, 1 ], [ 560, 3 ], [ 565, 4 ], [ 570, 4 ], [ 575, 2 ] ], "l:269": [ [ 451, 17 ] ], "l:277": [ [ 156, 17 ] ], "l:279": [ [ 29, 4 ], [ 34, 1 ], [ 36, 4 ], [ 190, 8 ] ], "l:288": [ [ 368, 3 ] ], "l:290": [ [ 371, 3 ] ], "l:298": [ [ 49, 2 ], [ 240, 7 ], [ 248, 3 ], [ 252, 4 ], [ 335, 1 ] ], "l:314": [ [ 46, 1 ], [ 198, 9 ], [ 224, 7 ] ], "l:330": [ [ 51, 1 ], [ 55, 1 ], [ 67, 1 ], [ 258, 6 ], [ 266, 6 ], [ 336, 1 ], [ 391, 1 ] ], "l:34": [ [ 139, 17 ] ], "l:341": [ [ 657, 17 ] ], "l:358": [ [ 674, 17 ] ], "l:361": [ [ 374, 3 ] ], "l:386": [ [ 57, 1 ], [ 273, 12 ], [ 288, 2 ], [ 337, 1 ], [ 339, 1 ] ], "l:393": [ [ 18, 2 ], [ 329, 1 ], [ 344, 2 ], [ 468, 4 ], [ 490, 8 ] ], "l:395": [ [ 33, 1 ], [ 327, 1 ] ], "l:398": [ [ 20, 1 ], [ 102, 1 ], [ 346, 1 ], [ 498, 4 ], [ 505, 6 ], [ 512, 3 ], [ 519, 1 ] ], "l:400": [ [ 68, 1 ], [ 356, 1 ] ], "l:401": [ [ 364, 1 ] ], "l:408": [ [ 726, 17 ] ], "l:413": [ [ 21, 1 ], [ 103, 3 ], [ 515, 4 ], [ 520, 2 ], [ 525, 7 ] ], "l:418": [ [ 22, 2 ], [ 106, 5 ], [ 120, 1 ], [ 347, 1 ], [ 395, 1 ], [ 532, 2 ], [ 535, 3 ], [ 540, 2 ] ], "l:426": [ [ 286, 1 ] ], "l:434": [ [ 27, 1 ], [ 69, 16 ] ], "l:447": [ [ 231, 1 ] ], "l:455": [ [ 85, 17 ] ], "l:465": [ [ 232, 1 ] ], "l:475": [ [ 233, 1 ] ], "l:485": [ [ 287, 1 ] ], "l:495": [ [ 234, 1 ] ], "l:505": [ [ 235, 1 ] ], "l:60": [ [ 28, 1 ], [ 121, 16 ] ], "l:62": [ [ 35, 1 ] ], "l:71": [ [ 40, 1 ] ], "l:79": [ [ 472, 17 ] ] } }, { "key": "L17.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:14": [ [ 285, 1 ] ], "l:179": [ [ 341, 1 ], [ 343, 1 ], [ 377, 12 ], [ 390, 1 ], [ 394, 1 ], [ 398, 1 ] ], "l:2": [ [ 26, 1 ] ], "l:205": [ [ 44, 1 ], [ 220, 1 ], [ 222, 1 ] ], "l:232": [ [ 188, 1 ], [ 327, 1 ], [ 333, 1 ] ], "l:262": [ [ 434, 17 ] ], "l:265": [ [ 417, 17 ] ], "l:268": [ [ 112, 3 ], [ 348, 1 ], [ 560, 3 ], [ 565, 4 ], [ 570, 4 ], [ 575, 2 ] ], "l:269": [ [ 451, 17 ] ], "l:277": [ [ 156, 17 ] ], "l:279": [ [ 29, 4 ], [ 34, 1 ], [ 36, 4 ], [ 190, 8 ] ], "l:286": [ [ 207, 1 ], [ 209, 1 ], [ 211, 1 ] ], "l:298": [ [ 49, 2 ], [ 240, 7 ], [ 248, 3 ], [ 252, 4 ], [ 335, 1 ] ], "l:314": [ [ 46, 1 ], [ 198, 9 ], [ 224, 7 ] ], "l:321": [ [ 208, 1 ], [ 334, 1 ], [ 404, 1 ] ], "l:330": [ [ 51, 1 ], [ 55, 1 ], [ 67, 1 ], [ 258, 6 ], [ 266, 6 ], [ 336, 1 ], [ 391, 1 ] ], "l:339": [ [ 42, 1 ], [ 45, 1 ], [ 210, 1 ], [ 212, 8 ], [ 223, 1 ], [ 236, 4 ], [ 256, 1 ] ], "l:34": [ [ 139, 17 ] ], "l:360": [ [ 65, 1 ], [ 326, 1 ] ], "l:377": [ [ 52, 3 ], [ 56, 1 ], [ 58, 2 ], [ 264, 2 ], [ 290, 9 ] ], "l:379": [ [ 43, 1 ], [ 60, 1 ] ], "l:386": [ [ 57, 1 ], [ 273, 12 ], [ 288, 2 ], [ 337, 1 ], [ 339, 1 ] ], "l:391": [ [ 61, 2 ], [ 299, 5 ], [ 305, 2 ], [ 308, 4 ], [ 313, 1 ], [ 315, 1 ], [ 338, 1 ], [ 393, 1 ] ], "l:396": [ [ 63, 2 ], [ 68, 1 ], [ 316, 1 ], [ 318, 1 ], [ 320, 3 ], [ 328, 1 ], [ 351, 1 ], [ 354, 1 ], [ 356, 6 ] ], "l:401": [ [ 364, 1 ] ], "l:406": [ [ 33, 1 ], [ 173, 15 ], [ 392, 1 ] ], "l:426": [ [ 286, 1 ] ], "l:434": [ [ 27, 1 ], [ 69, 16 ] ], "l:447": [ [ 231, 1 ] ], "l:455": [ [ 85, 17 ] ], "l:465": [ [ 232, 1 ] ], "l:475": [ [ 233, 1 ] ], "l:485": [ [ 287, 1 ] ], "l:495": [ [ 234, 1 ] ], "l:505": [ [ 235, 1 ] ], "l:60": [ [ 28, 1 ], [ 121, 16 ] ], "l:62": [ [ 35, 1 ] ], "l:71": [ [ 40, 1 ] ], "l:79": [ [ 472, 17 ] ] } }, { "key": "L18.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:14": [ [ 285, 1 ] ], "l:179": [ [ 341, 1 ], [ 343, 1 ], [ 377, 12 ], [ 390, 1 ], [ 394, 1 ], [ 398, 1 ] ], "l:2": [ [ 26, 1 ] ], "l:203": [ [ 299, 3 ] ], "l:277": [ [ 156, 17 ] ], "l:279": [ [ 29, 4 ], [ 34, 1 ], [ 36, 4 ], [ 190, 8 ] ], "l:283": [ [ 365, 12 ], [ 399, 5 ] ], "l:285": [ [ 43, 1 ], [ 273, 1 ] ], "l:298": [ [ 49, 2 ], [ 240, 7 ], [ 248, 3 ], [ 252, 4 ], [ 335, 1 ] ], "l:314": [ [ 46, 1 ], [ 198, 9 ], [ 224, 7 ] ], "l:318": [ [ 60, 3 ], [ 303, 1 ], [ 305, 2 ], [ 308, 4 ], [ 313, 1 ], [ 315, 1 ], [ 338, 1 ], [ 340, 1 ], [ 362, 2 ], [ 393, 1 ] ], "l:320": [ [ 65, 1 ], [ 326, 1 ] ], "l:330": [ [ 51, 1 ], [ 55, 1 ], [ 67, 1 ], [ 258, 6 ], [ 266, 6 ], [ 336, 1 ], [ 391, 1 ] ], "l:337": [ [ 18, 1 ], [ 329, 1 ], [ 342, 1 ], [ 344, 1 ], [ 405, 9 ], [ 415, 2 ], [ 468, 2 ] ], "l:34": [ [ 139, 17 ] ], "l:375": [ [ 19, 1 ], [ 345, 1 ], [ 470, 2 ], [ 490, 12 ], [ 510, 1 ] ], "l:384": [ [ 57, 1 ], [ 274, 11 ], [ 288, 2 ], [ 302, 1 ], [ 337, 1 ], [ 339, 1 ] ], "l:401": [ [ 364, 1 ] ], "l:406": [ [ 33, 1 ], [ 173, 15 ], [ 392, 1 ] ], "l:426": [ [ 286, 1 ] ], "l:434": [ [ 27, 1 ], [ 69, 16 ] ], "l:447": [ [ 231, 1 ] ], "l:455": [ [ 85, 17 ] ], "l:465": [ [ 232, 1 ] ], "l:475": [ [ 233, 1 ] ], "l:485": [ [ 287, 1 ] ], "l:495": [ [ 234, 1 ] ], "l:505": [ [ 235, 1 ] ], "l:60": [ [ 28, 1 ], [ 121, 16 ] ], "l:71": [ [ 40, 1 ] ], "l:79": [ [ 472, 17 ] ] } }, { "key": "L19.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:14": [ [ 285, 1 ] ], "l:2": [ [ 26, 1 ] ], "l:202": [ [ 65, 1 ], [ 326, 1 ] ], "l:277": [ [ 156, 17 ] ], "l:279": [ [ 29, 4 ], [ 34, 1 ], [ 36, 4 ], [ 190, 8 ] ], "l:281": [ [ 52, 4 ], [ 67, 1 ], [ 258, 4 ], [ 264, 3 ], [ 268, 3 ], [ 273, 1 ], [ 391, 1 ] ], "l:298": [ [ 49, 2 ], [ 240, 7 ], [ 248, 3 ], [ 252, 4 ], [ 335, 1 ] ], "l:314": [ [ 46, 1 ], [ 198, 9 ], [ 224, 7 ] ], "l:316": [ [ 56, 1 ], [ 58, 2 ], [ 63, 1 ], [ 290, 9 ], [ 316, 1 ], [ 318, 1 ], [ 320, 1 ], [ 328, 1 ] ], "l:327": [ [ 321, 1 ] ], "l:328": [ [ 35, 1 ], [ 42, 2 ], [ 188, 1 ], [ 207, 9 ], [ 327, 1 ], [ 333, 2 ], [ 404, 1 ] ], "l:34": [ [ 139, 17 ] ], "l:384": [ [ 57, 1 ], [ 274, 11 ], [ 288, 2 ], [ 302, 1 ], [ 337, 1 ], [ 339, 1 ] ], "l:401": [ [ 364, 1 ] ], "l:406": [ [ 33, 1 ], [ 173, 15 ], [ 392, 1 ] ], "l:426": [ [ 286, 1 ] ], "l:434": [ [ 27, 1 ], [ 69, 16 ] ], "l:447": [ [ 231, 1 ] ], "l:455": [ [ 85, 17 ] ], "l:465": [ [ 232, 1 ] ], "l:475": [ [ 233, 1 ] ], "l:485": [ [ 287, 1 ] ], "l:495": [ [ 234, 1 ] ], "l:505": [ [ 235, 1 ] ], "l:60": [ [ 28, 1 ], [ 121, 16 ] ], "l:71": [ [ 40, 1 ] ], "l:77": [ [ 44, 2 ], [ 51, 1 ], [ 216, 5 ], [ 222, 2 ], [ 236, 4 ], [ 256, 1 ], [ 262, 2 ] ] } }, { "key": "L20.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:14": [ [ 285, 1 ] ], "l:2": [ [ 26, 1 ] ], "l:274": [ [ 50, 1 ] ], "l:275": [ [ 18, 1 ], [ 29, 4 ], [ 34, 1 ], [ 65, 1 ], [ 156, 8 ], [ 326, 1 ], [ 329, 1 ] ], "l:295": [ [ 253, 1 ] ], "l:296": [ [ 36, 3 ], [ 164, 9 ], [ 190, 4 ], [ 197, 1 ] ], "l:326": [ [ 254, 1 ] ], "l:328": [ [ 35, 1 ], [ 42, 2 ], [ 188, 1 ], [ 207, 9 ], [ 327, 1 ], [ 333, 2 ], [ 404, 1 ] ], "l:34": [ [ 139, 17 ] ], "l:384": [ [ 57, 1 ], [ 274, 11 ], [ 288, 2 ], [ 302, 1 ], [ 337, 1 ], [ 339, 1 ] ], "l:401": [ [ 364, 1 ] ], "l:406": [ [ 33, 1 ], [ 173, 15 ], [ 392, 1 ] ], "l:426": [ [ 286, 1 ] ], "l:434": [ [ 27, 1 ], [ 69, 16 ] ], "l:447": [ [ 231, 1 ] ], "l:455": [ [ 85, 17 ] ], "l:465": [ [ 232, 1 ] ], "l:475": [ [ 233, 1 ] ], "l:485": [ [ 287, 1 ] ], "l:495": [ [ 234, 1 ] ], "l:505": [ [ 235, 1 ] ], "l:60": [ [ 28, 1 ], [ 121, 16 ] ], "l:69": [ [ 39, 1 ], [ 49, 1 ], [ 194, 3 ], [ 240, 7 ], [ 248, 3 ], [ 252, 1 ], [ 335, 1 ] ] } }, { "key": "L21.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:14": [ [ 285, 1 ] ], "l:2": [ [ 26, 1 ] ], "l:273": [ [ 225, 1 ] ], "l:275": [ [ 18, 1 ], [ 29, 4 ], [ 34, 1 ], [ 65, 1 ], [ 156, 8 ], [ 326, 1 ], [ 329, 1 ] ], "l:294": [ [ 226, 1 ] ], "l:296": [ [ 36, 3 ], [ 164, 9 ], [ 190, 4 ], [ 197, 1 ] ], "l:31": [ [ 223, 1 ] ], "l:32": [ [ 40, 1 ], [ 44, 1 ], [ 198, 9 ], [ 216, 5 ], [ 222, 1 ] ], "l:326": [ [ 254, 1 ] ], "l:328": [ [ 35, 1 ], [ 42, 2 ], [ 188, 1 ], [ 207, 9 ], [ 327, 1 ], [ 333, 2 ], [ 404, 1 ] ], "l:383": [ [ 224, 1 ] ], "l:384": [ [ 57, 1 ], [ 274, 11 ], [ 288, 2 ], [ 302, 1 ], [ 337, 1 ], [ 339, 1 ] ], "l:401": [ [ 364, 1 ] ], "l:406": [ [ 33, 1 ], [ 173, 15 ], [ 392, 1 ] ], "l:426": [ [ 286, 1 ] ], "l:434": [ [ 27, 1 ], [ 69, 16 ] ], "l:447": [ [ 231, 1 ] ], "l:455": [ [ 85, 17 ] ], "l:465": [ [ 232, 1 ] ], "l:475": [ [ 233, 1 ] ], "l:485": [ [ 287, 1 ] ], "l:495": [ [ 234, 1 ] ], "l:505": [ [ 235, 1 ] ] } }, { "key": "L22.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:14": [ [ 285, 1 ] ], "l:2": [ [ 26, 1 ] ], "l:270": [ [ 122, 1 ] ], "l:272": [ [ 121, 1 ] ], "l:275": [ [ 18, 1 ], [ 29, 4 ], [ 34, 1 ], [ 65, 1 ], [ 156, 8 ], [ 326, 1 ], [ 329, 1 ] ], "l:292": [ [ 123, 1 ] ], "l:296": [ [ 36, 3 ], [ 164, 9 ], [ 190, 4 ], [ 197, 1 ] ], "l:30": [ [ 124, 1 ] ], "l:32": [ [ 40, 1 ], [ 44, 1 ], [ 198, 9 ], [ 216, 5 ], [ 222, 1 ] ], "l:324": [ [ 125, 1 ] ], "l:328": [ [ 35, 1 ], [ 42, 2 ], [ 188, 1 ], [ 207, 9 ], [ 327, 1 ], [ 333, 2 ], [ 404, 1 ] ], "l:381": [ [ 126, 1 ] ], "l:384": [ [ 57, 1 ], [ 274, 11 ], [ 288, 2 ], [ 302, 1 ], [ 337, 1 ], [ 339, 1 ] ], "l:401": [ [ 364, 1 ] ], "l:406": [ [ 33, 1 ], [ 173, 15 ], [ 392, 1 ] ], "l:426": [ [ 286, 1 ] ], "l:434": [ [ 27, 1 ], [ 69, 16 ] ], "l:447": [ [ 231, 1 ] ], "l:455": [ [ 85, 17 ] ], "l:465": [ [ 232, 1 ] ], "l:475": [ [ 233, 1 ] ], "l:485": [ [ 287, 1 ] ], "l:495": [ [ 234, 1 ] ], "l:505": [ [ 235, 1 ] ] } }, { "key": "L23.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:14": [ [ 285, 1 ] ], "l:2": [ [ 26, 1 ] ], "l:270": [ [ 122, 1 ] ], "l:275": [ [ 18, 1 ], [ 29, 4 ], [ 34, 1 ], [ 65, 1 ], [ 156, 8 ], [ 326, 1 ], [ 329, 1 ] ], "l:292": [ [ 123, 1 ] ], "l:296": [ [ 36, 3 ], [ 164, 9 ], [ 190, 4 ], [ 197, 1 ] ], "l:30": [ [ 124, 1 ] ], "l:32": [ [ 40, 1 ], [ 44, 1 ], [ 198, 9 ], [ 216, 5 ], [ 222, 1 ] ], "l:323": [ [ 121, 1 ] ], "l:328": [ [ 35, 1 ], [ 42, 2 ], [ 188, 1 ], [ 207, 9 ], [ 327, 1 ], [ 333, 2 ], [ 404, 1 ] ], "l:380": [ [ 127, 1 ] ], "l:384": [ [ 57, 1 ], [ 274, 11 ], [ 288, 2 ], [ 302, 1 ], [ 337, 1 ], [ 339, 1 ] ], "l:401": [ [ 364, 1 ] ], "l:406": [ [ 33, 1 ], [ 173, 15 ], [ 392, 1 ] ], "l:426": [ [ 286, 1 ] ], "l:434": [ [ 27, 1 ], [ 69, 16 ] ], "l:447": [ [ 231, 1 ] ], "l:455": [ [ 85, 17 ] ], "l:465": [ [ 232, 1 ] ], "l:475": [ [ 233, 1 ] ], "l:485": [ [ 287, 1 ] ], "l:495": [ [ 234, 1 ] ], "l:505": [ [ 235, 1 ] ] } }, { "key": "L24.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:14": [ [ 285, 1 ] ], "l:2": [ [ 26, 1 ] ], "l:27": [ [ 121, 1 ] ], "l:275": [ [ 18, 1 ], [ 29, 4 ], [ 34, 1 ], [ 65, 1 ], [ 156, 8 ], [ 326, 1 ], [ 329, 1 ] ], "l:296": [ [ 36, 3 ], [ 164, 9 ], [ 190, 4 ], [ 197, 1 ] ], "l:32": [ [ 40, 1 ], [ 44, 1 ], [ 198, 9 ], [ 216, 5 ], [ 222, 1 ] ], "l:328": [ [ 35, 1 ], [ 42, 2 ], [ 188, 1 ], [ 207, 9 ], [ 327, 1 ], [ 333, 2 ], [ 404, 1 ] ], "l:384": [ [ 57, 1 ], [ 274, 11 ], [ 288, 2 ], [ 302, 1 ], [ 337, 1 ], [ 339, 1 ] ], "l:401": [ [ 364, 1 ] ], "l:406": [ [ 33, 1 ], [ 173, 15 ], [ 392, 1 ] ], "l:426": [ [ 286, 1 ] ], "l:434": [ [ 27, 1 ], [ 69, 16 ] ], "l:447": [ [ 231, 1 ] ], "l:455": [ [ 85, 17 ] ], "l:465": [ [ 232, 1 ] ], "l:475": [ [ 233, 1 ] ], "l:485": [ [ 287, 1 ] ], "l:495": [ [ 234, 1 ] ], "l:505": [ [ 235, 1 ] ] } }, { "key": "L25.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:14": [ [ 285, 1 ] ], "l:2": [ [ 26, 1 ] ], "l:26": [ [ 122, 12 ] ], "l:27": [ [ 121, 1 ] ], "l:275": [ [ 18, 1 ], [ 29, 4 ], [ 34, 1 ], [ 65, 1 ], [ 156, 8 ], [ 326, 1 ], [ 329, 1 ] ], "l:296": [ [ 36, 3 ], [ 164, 9 ], [ 190, 4 ], [ 197, 1 ] ], "l:32": [ [ 40, 1 ], [ 44, 1 ], [ 198, 9 ], [ 216, 5 ], [ 222, 1 ] ], "l:328": [ [ 35, 1 ], [ 42, 2 ], [ 188, 1 ], [ 207, 9 ], [ 327, 1 ], [ 333, 2 ], [ 404, 1 ] ], "l:384": [ [ 57, 1 ], [ 274, 11 ], [ 288, 2 ], [ 302, 1 ], [ 337, 1 ], [ 339, 1 ] ], "l:406": [ [ 33, 1 ], [ 173, 15 ], [ 392, 1 ] ], "l:423": [ [ 28, 1 ], [ 134, 3 ], [ 139, 13 ] ], "l:426": [ [ 286, 1 ] ], "l:434": [ [ 27, 1 ], [ 69, 16 ] ], "l:447": [ [ 231, 1 ] ], "l:455": [ [ 85, 17 ] ], "l:465": [ [ 232, 1 ] ], "l:475": [ [ 233, 1 ] ], "l:485": [ [ 287, 1 ] ], "l:495": [ [ 234, 1 ] ], "l:505": [ [ 235, 1 ] ] } }, { "key": "L26.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:14": [ [ 285, 1 ] ], "l:2": [ [ 26, 1 ] ], "l:24": [ [ 39, 1 ], [ 45, 2 ], [ 49, 8 ], [ 58, 7 ], [ 67, 2 ], [ 121, 1 ], [ 152, 4 ], [ 194, 3 ], [ 223, 4 ], [ 228, 3 ], [ 236, 11 ], [ 248, 3 ], [ 252, 2 ], [ 255, 2 ], [ 258, 6 ], [ 265, 2 ], [ 268, 3 ], [ 273, 1 ], [ 290, 12 ], [ 303, 1 ], [ 305, 2 ], [ 308, 4 ], [ 313, 1 ], [ 315, 2 ], [ 318, 1 ], [ 320, 3 ], [ 328, 1 ], [ 335, 2 ], [ 338, 1 ], [ 340, 5 ], [ 351, 1 ], [ 354, 1 ], [ 356, 27 ], [ 384, 5 ], [ 390, 2 ], [ 393, 2 ], [ 398, 6 ], [ 405, 9 ], [ 415, 51 ] ], "l:426": [ [ 286, 1 ] ], "l:431": [ [ 19, 7 ], [ 102, 19 ], [ 345, 6 ], [ 352, 1 ], [ 395, 1 ], [ 466, 13 ], [ 480, 9 ], [ 490, 12 ], [ 505, 6 ], [ 512, 10 ], [ 525, 9 ], [ 535, 3 ], [ 540, 3 ], [ 544, 5 ], [ 550, 3 ], [ 555, 1 ], [ 557, 2 ], [ 560, 3 ], [ 565, 4 ], [ 570, 4 ], [ 576, 3 ], [ 580, 4 ], [ 585, 3 ], [ 601, 30 ], [ 632, 37 ], [ 670, 6 ] ], "l:447": [ [ 231, 1 ] ], "l:452": [ [ 66, 1 ], [ 137, 2 ], [ 331, 2 ], [ 353, 1 ], [ 355, 1 ], [ 396, 2 ], [ 676, 3 ], [ 680, 6 ], [ 691, 44 ], [ 736, 68 ], [ 805, 16 ], [ 822, 33 ], [ 864, 25 ] ], "l:465": [ [ 232, 1 ] ], "l:470": [ [ 889, 204 ] ], "l:475": [ [ 233, 1 ] ], "l:480": [ [ 1093, 204 ] ], "l:485": [ [ 287, 1 ] ], "l:490": [ [ 1297, 204 ] ], "l:495": [ [ 234, 1 ] ], "l:500": [ [ 1501, 204 ] ], "l:505": [ [ 235, 1 ] ], "l:510": [ [ 1705, 204 ] ] } }, { "key": "L27.post", "nodes": { "i:0": [ [ 0, 1 ] ], "l:12": [ [ 44, 1 ], [ 74, 3 ], [ 80, 1 ], [ 217, 4 ], [ 222, 1 ], [ 274, 7 ] ], "l:2": [ [ 26, 1 ] ], "l:23": [ [ 18, 1 ], [ 65, 1 ], [ 81, 13 ], [ 326, 1 ], [ 329, 1 ] ], "l:424": [ [ 57, 1 ], [ 77, 2 ], [ 281, 4 ], [ 288, 2 ], [ 302, 1 ], [ 337, 1 ], [ 339, 1 ], [ 1909, 5 ] ], "l:430": [ [ 69, 2 ], [ 79, 1 ], [ 94, 8 ], [ 122, 6 ] ], "l:445": [ [ 1914, 17 ] ], "l:451": [ [ 27, 2 ], [ 128, 9 ], [ 139, 6 ] ], "l:463": [ [ 1931, 17 ] ], "l:465": [ [ 232, 1 ] ], "l:469": [ [ 29, 3 ], [ 71, 1 ], [ 145, 7 ], [ 156, 6 ] ], "l:473": [ [ 1948, 17 ] ], "l:479": [ [ 32, 3 ], [ 162, 14 ] ], "l:483": [ [ 1965, 17 ] ], "l:489": [ [ 35, 3 ], [ 72, 1 ], [ 176, 12 ], [ 392, 1 ] ], "l:493": [ [ 1982, 17 ] ], "l:499": [ [ 38, 1 ], [ 40, 1 ], [ 73, 1 ], [ 188, 1 ], [ 190, 4 ], [ 197, 6 ], [ 327, 1 ], [ 333, 1 ], [ 404, 1 ] ], "l:503": [ [ 1999, 17 ] ], "l:509": [ [ 42, 2 ], [ 203, 14 ], [ 334, 1 ] ], "l:513": [ [ 2016, 17 ] ] } }, { "key": "L28.post", "nodes": { "l:0": [ [ 1, 17 ] ], "l:10": [ [ 18, 1 ], [ 27, 1 ], [ 65, 1 ], [ 82, 8 ], [ 91, 2 ], [ 329, 1 ], [ 1631, 1 ], [ 1645, 2 ] ], "l:2": [ [ 26, 1 ] ], "l:513": [ [ 2016, 17 ] ] } }, { "key": "output", "nodes": { "l:0": [ [ 1, 17 ] ] } } ], "width": 2048 }, "runtime_contract": { "causal_attention": true, "dtype": "float32", "generation": { "do_sample": false, "greedy": true }, "recommended_attention_implementation_for_analysis": "eager", "unsupported_inferences": [ "Static graph ancestry is not dynamic causal relevance.", "Unused-on-one-input activations require an input-specific trace.", "Other numerical precisions are outside the exactness contract." ] }, "schedule": { "assignment": [ { "cancel_layer": 7, "cancel_sublayer": "attn", "layer": null, "node": "l:8", "routing": null }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 0, "node": "l:46", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 0, "node": "l:48", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 0, "node": "l:50", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "mlp", "layer": 1, "node": "l:45", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "mlp", "layer": 1, "node": "l:47", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "mlp", "layer": 1, "node": "l:49", "routing": "mlp" }, { "cancel_layer": 3, "cancel_sublayer": "mlp", "layer": 2, "node": "l:43", "routing": "mlp" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 0, "node": "l:7", "routing": "mlp" }, { "cancel_layer": 1, "cancel_sublayer": "attn", "layer": 0, "node": "l:54", "routing": "mlp" }, { "cancel_layer": 1, "cancel_sublayer": "attn", "layer": 0, "node": "l:55", "routing": "mlp" }, { "cancel_layer": 1, "cancel_sublayer": "attn", "layer": 0, "node": "l:57", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:52", "routing": "attn" }, { "cancel_layer": 3, "cancel_sublayer": "mlp", "layer": 1, "node": "l:51", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 3, "node": "l:41", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 3, "node": "l:38", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 3, "node": "l:39", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 3, "node": "l:58", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "mlp", "layer": 4, "node": "l:36", "routing": "attn" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 3, "node": "l:64", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 3, "node": "l:65", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 3, "node": "l:67", "routing": "mlp" }, { "cancel_layer": 18, "cancel_sublayer": "mlp", "layer": 4, "node": "l:62", "routing": "attn" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 3, "node": "l:73", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 3, "node": "l:74", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 3, "node": "l:76", "routing": "mlp" }, { "cancel_layer": 20, "cancel_sublayer": "mlp", "layer": 4, "node": "l:71", "routing": "attn" }, { "cancel_layer": 1, "cancel_sublayer": "mlp", "layer": 0, "node": "l:106", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 3, "node": "l:119", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 3, "node": "l:120", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 3, "node": "l:122", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 4, "node": "l:117", "routing": "attn" }, { "cancel_layer": 5, "cancel_sublayer": "mlp", "layer": 0, "node": "l:115", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "mlp", "layer": 4, "node": "l:116", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "mlp", "layer": 5, "node": "l:113", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 1, "node": "l:166", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "mlp", "layer": 4, "node": "l:164", "routing": "mlp" }, { "cancel_layer": 1, "cancel_sublayer": "attn", "layer": 0, "node": "l:103", "routing": "mlp" }, { "cancel_layer": 1, "cancel_sublayer": "attn", "layer": 0, "node": "l:104", "routing": "mlp" }, { "cancel_layer": 1, "cancel_sublayer": "attn", "layer": 0, "node": "l:107", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:101", "routing": "attn" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:108", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 0, "node": "l:100", "routing": "mlp" }, { "cancel_layer": 3, "cancel_sublayer": "attn", "layer": 2, "node": "l:99", "routing": "attn" }, { "cancel_layer": 3, "cancel_sublayer": "mlp", "layer": 2, "node": "l:98", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 3, "node": "l:97", "routing": "attn" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 3, "node": "l:126", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 4, "node": "l:127", "routing": "attn" }, { "cancel_layer": 5, "cancel_sublayer": "mlp", "layer": 4, "node": "l:124", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "mlp", "layer": 5, "node": "l:123", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 6, "node": "l:111", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 6, "node": "l:128", "routing": "mlp" }, { "cancel_layer": 1, "cancel_sublayer": "mlp", "layer": 0, "node": "l:131", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 1, "node": "l:129", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 7, "node": "l:109", "routing": "attn" }, { "cancel_layer": 6, "cancel_sublayer": "mlp", "layer": 5, "node": "l:134", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 6, "node": "l:132", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 0, "node": "l:155", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 0, "node": "l:156", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 0, "node": "l:158", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 4, "node": "l:153", "routing": "attn" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 4, "node": "l:159", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 0, "node": "l:152", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 5, "node": "l:151", "routing": "attn" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 5, "node": "l:150", "routing": "mlp" }, { "cancel_layer": 7, "cancel_sublayer": "attn", "layer": 6, "node": "l:149", "routing": "attn" }, { "cancel_layer": 7, "cancel_sublayer": "mlp", "layer": 6, "node": "l:170", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 7, "node": "l:171", "routing": "attn" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 7, "node": "l:168", "routing": "mlp" }, { "cancel_layer": 9, "cancel_sublayer": "attn", "layer": 8, "node": "l:167", "routing": "attn" }, { "cancel_layer": 10, "cancel_sublayer": "attn", "layer": 8, "node": "l:162", "routing": "mlp" }, { "cancel_layer": 10, "cancel_sublayer": "mlp", "layer": 8, "node": "l:172", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 3, "node": "l:175", "routing": "mlp" }, { "cancel_layer": 10, "cancel_sublayer": "attn", "layer": 4, "node": "l:173", "routing": "mlp" }, { "cancel_layer": 10, "cancel_sublayer": "attn", "layer": 9, "node": "l:160", "routing": "attn" }, { "cancel_layer": 5, "cancel_sublayer": "mlp", "layer": 4, "node": "l:178", "routing": "mlp" }, { "cancel_layer": 10, "cancel_sublayer": "attn", "layer": 5, "node": "l:176", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 3, "node": "l:93", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 3, "node": "l:94", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "attn", "layer": 3, "node": "l:96", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 4, "node": "l:91", "routing": "attn" }, { "cancel_layer": 7, "cancel_sublayer": "attn", "layer": 6, "node": "l:145", "routing": "mlp" }, { "cancel_layer": 7, "cancel_sublayer": "attn", "layer": 6, "node": "l:146", "routing": "mlp" }, { "cancel_layer": 7, "cancel_sublayer": "attn", "layer": 6, "node": "l:148", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 7, "node": "l:143", "routing": "attn" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 8, "node": "l:89", "routing": "attn" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 10, "node": "l:85", "routing": "mlp" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 8, "node": "l:86", "routing": "mlp" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 10, "node": "l:88", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "mlp", "layer": 11, "node": "l:83", "routing": "attn" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 8, "node": "l:200", "routing": "attn" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 10, "node": "l:196", "routing": "mlp" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 10, "node": "l:197", "routing": "mlp" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 10, "node": "l:199", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "mlp", "layer": 11, "node": "l:194", "routing": "attn" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 8, "node": "l:220", "routing": "attn" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 10, "node": "l:216", "routing": "mlp" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 10, "node": "l:217", "routing": "mlp" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 10, "node": "l:219", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "mlp", "layer": 11, "node": "l:214", "routing": "attn" }, { "cancel_layer": 11, "cancel_sublayer": "mlp", "layer": 10, "node": "l:141", "routing": "attn" }, { "cancel_layer": 11, "cancel_sublayer": "mlp", "layer": 10, "node": "l:137", "routing": "mlp" }, { "cancel_layer": 11, "cancel_sublayer": "mlp", "layer": 10, "node": "l:138", "routing": "mlp" }, { "cancel_layer": 11, "cancel_sublayer": "mlp", "layer": 10, "node": "l:140", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "mlp", "layer": 11, "node": "l:135", "routing": "attn" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 10, "node": "l:190", "routing": "attn" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 10, "node": "l:186", "routing": "mlp" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 10, "node": "l:187", "routing": "mlp" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 10, "node": "l:189", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "mlp", "layer": 11, "node": "l:184", "routing": "attn" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 10, "node": "l:228", "routing": "attn" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 10, "node": "l:224", "routing": "mlp" }, { "cancel_layer": 11, "cancel_sublayer": "mlp", "layer": 10, "node": "l:225", "routing": "mlp" }, { "cancel_layer": 11, "cancel_sublayer": "attn", "layer": 10, "node": "l:227", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "mlp", "layer": 11, "node": "l:222", "routing": "attn" }, { "cancel_layer": 14, "cancel_sublayer": "mlp", "layer": 13, "node": "l:302", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "mlp", "layer": 11, "node": "l:306", "routing": "mlp" }, { "cancel_layer": 12, "cancel_sublayer": "mlp", "layer": 11, "node": "l:310", "routing": "mlp" }, { "cancel_layer": 12, "cancel_sublayer": "mlp", "layer": 11, "node": "l:312", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "mlp", "layer": 12, "node": "l:335", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "mlp", "layer": 12, "node": "l:308", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "mlp", "layer": 13, "node": "l:304", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "attn", "layer": 12, "node": "l:412", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "mlp", "layer": 12, "node": "l:390", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "mlp", "layer": 13, "node": "l:334", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "mlp", "layer": 14, "node": "l:303", "routing": "attn" }, { "cancel_layer": 14, "cancel_sublayer": "mlp", "layer": 13, "node": "l:410", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "mlp", "layer": 13, "node": "l:388", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "mlp", "layer": 14, "node": "l:332", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "mlp", "layer": 14, "node": "l:300", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 3, "node": "l:59", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 3, "node": "l:438", "routing": "mlp" }, { "cancel_layer": 12, "cancel_sublayer": "mlp", "layer": 11, "node": "l:347", "routing": "mlp" }, { "cancel_layer": 12, "cancel_sublayer": "mlp", "layer": 11, "node": "l:351", "routing": "mlp" }, { "cancel_layer": 12, "cancel_sublayer": "mlp", "layer": 11, "node": "l:355", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "mlp", "layer": 12, "node": "l:346", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "attn", "layer": 12, "node": "l:349", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "mlp", "layer": 12, "node": "l:350", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "attn", "layer": 12, "node": "l:353", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "mlp", "layer": 12, "node": "l:354", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "attn", "layer": 12, "node": "l:357", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "mlp", "layer": 13, "node": "l:344", "routing": "attn" }, { "cancel_layer": 15, "cancel_sublayer": "attn", "layer": 13, "node": "l:343", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "mlp", "layer": 14, "node": "l:341", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "mlp", "layer": 14, "node": "l:371", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "mlp", "layer": 14, "node": "l:365", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "mlp", "layer": 14, "node": "l:358", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "mlp", "layer": 14, "node": "l:373", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "mlp", "layer": 14, "node": "l:367", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "mlp", "layer": 15, "node": "l:363", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "mlp", "layer": 15, "node": "l:369", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "mlp", "layer": 16, "node": "l:361", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "mlp", "layer": 14, "node": "l:417", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 16, "node": "l:395", "routing": "attn" }, { "cancel_layer": 18, "cancel_sublayer": "attn", "layer": 17, "node": "l:360", "routing": "attn" }, { "cancel_layer": 16, "cancel_sublayer": "mlp", "layer": 15, "node": "l:415", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "mlp", "layer": 16, "node": "l:393", "routing": "mlp" }, { "cancel_layer": 18, "cancel_sublayer": "mlp", "layer": 17, "node": "l:339", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "mlp", "layer": 4, "node": "l:68", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 3, "node": "l:441", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "mlp", "layer": 14, "node": "l:231", "routing": "mlp" }, { "cancel_layer": 12, "cancel_sublayer": "mlp", "layer": 11, "node": "l:81", "routing": "mlp" }, { "cancel_layer": 12, "cancel_sublayer": "attn", "layer": 11, "node": "l:182", "routing": "mlp" }, { "cancel_layer": 12, "cancel_sublayer": "mlp", "layer": 11, "node": "l:251", "routing": "mlp" }, { "cancel_layer": 12, "cancel_sublayer": "attn", "layer": 11, "node": "l:192", "routing": "mlp" }, { "cancel_layer": 12, "cancel_sublayer": "attn", "layer": 11, "node": "l:253", "routing": "mlp" }, { "cancel_layer": 12, "cancel_sublayer": "mlp", "layer": 11, "node": "l:241", "routing": "mlp" }, { "cancel_layer": 12, "cancel_sublayer": "attn", "layer": 11, "node": "l:255", "routing": "mlp" }, { "cancel_layer": 12, "cancel_sublayer": "mlp", "layer": 11, "node": "l:243", "routing": "mlp" }, { "cancel_layer": 12, "cancel_sublayer": "mlp", "layer": 11, "node": "l:212", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "mlp", "layer": 12, "node": "l:211", "routing": "attn" }, { "cancel_layer": 13, "cancel_sublayer": "mlp", "layer": 12, "node": "l:230", "routing": "mlp" }, { "cancel_layer": 14, "cancel_sublayer": "mlp", "layer": 13, "node": "l:209", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "attn", "layer": 14, "node": "l:208", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "mlp", "layer": 15, "node": "l:207", "routing": "attn" }, { "cancel_layer": 16, "cancel_sublayer": "mlp", "layer": 15, "node": "l:245", "routing": "attn" }, { "cancel_layer": 15, "cancel_sublayer": "attn", "layer": 12, "node": "l:239", "routing": "mlp" }, { "cancel_layer": 13, "cancel_sublayer": "mlp", "layer": 12, "node": "l:249", "routing": "attn" }, { "cancel_layer": 13, "cancel_sublayer": "mlp", "layer": 12, "node": "l:263", "routing": "attn" }, { "cancel_layer": 16, "cancel_sublayer": "mlp", "layer": 12, "node": "l:180", "routing": "attn" }, { "cancel_layer": 13, "cancel_sublayer": "mlp", "layer": 12, "node": "l:80", "routing": "mlp" }, { "cancel_layer": 18, "cancel_sublayer": "mlp", "layer": 15, "node": "l:205", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "mlp", "layer": 15, "node": "l:236", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "mlp", "layer": 15, "node": "l:246", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "mlp", "layer": 14, "node": "l:260", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 15, "node": "l:266", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "mlp", "layer": 16, "node": "l:288", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "mlp", "layer": 16, "node": "l:234", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "mlp", "layer": 16, "node": "l:290", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "mlp", "layer": 16, "node": "l:258", "routing": "mlp" }, { "cancel_layer": 18, "cancel_sublayer": "mlp", "layer": 17, "node": "l:321", "routing": "mlp" }, { "cancel_layer": 18, "cancel_sublayer": "mlp", "layer": 17, "node": "l:286", "routing": "mlp" }, { "cancel_layer": 18, "cancel_sublayer": "mlp", "layer": 17, "node": "l:232", "routing": "mlp" }, { "cancel_layer": 19, "cancel_sublayer": "mlp", "layer": 18, "node": "l:203", "routing": "mlp" }, { "cancel_layer": 15, "cancel_sublayer": "mlp", "layer": 14, "node": "l:422", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 16, "node": "l:400", "routing": "attn" }, { "cancel_layer": 18, "cancel_sublayer": "attn", "layer": 17, "node": "l:379", "routing": "attn" }, { "cancel_layer": 19, "cancel_sublayer": "attn", "layer": 18, "node": "l:320", "routing": "attn" }, { "cancel_layer": 19, "cancel_sublayer": "attn", "layer": 18, "node": "l:285", "routing": "attn" }, { "cancel_layer": 20, "cancel_sublayer": "attn", "layer": 19, "node": "l:202", "routing": "attn" }, { "cancel_layer": 16, "cancel_sublayer": "mlp", "layer": 15, "node": "l:420", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "mlp", "layer": 16, "node": "l:398", "routing": "mlp" }, { "cancel_layer": 18, "cancel_sublayer": "mlp", "layer": 17, "node": "l:377", "routing": "mlp" }, { "cancel_layer": 19, "cancel_sublayer": "mlp", "layer": 18, "node": "l:318", "routing": "mlp" }, { "cancel_layer": 19, "cancel_sublayer": "mlp", "layer": 18, "node": "l:283", "routing": "mlp" }, { "cancel_layer": 20, "cancel_sublayer": "mlp", "layer": 19, "node": "l:77", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 3, "node": "l:444", "routing": "mlp" }, { "cancel_layer": 21, "cancel_sublayer": "mlp", "layer": 4, "node": "l:34", "routing": "mlp" }, { "cancel_layer": 21, "cancel_sublayer": "mlp", "layer": 5, "node": "l:60", "routing": "mlp" }, { "cancel_layer": 21, "cancel_sublayer": "mlp", "layer": 20, "node": "l:69", "routing": "mlp" }, { "cancel_layer": 26, "cancel_sublayer": "mlp", "layer": 21, "node": "l:32", "routing": "attn" }, { "cancel_layer": 20, "cancel_sublayer": "attn", "layer": 4, "node": "l:277", "routing": "mlp" }, { "cancel_layer": 20, "cancel_sublayer": "attn", "layer": 5, "node": "l:279", "routing": "mlp" }, { "cancel_layer": 20, "cancel_sublayer": "mlp", "layer": 19, "node": "l:281", "routing": "mlp" }, { "cancel_layer": 26, "cancel_sublayer": "mlp", "layer": 20, "node": "l:275", "routing": "attn" }, { "cancel_layer": 20, "cancel_sublayer": "attn", "layer": 15, "node": "l:298", "routing": "mlp" }, { "cancel_layer": 20, "cancel_sublayer": "mlp", "layer": 5, "node": "l:314", "routing": "mlp" }, { "cancel_layer": 20, "cancel_sublayer": "attn", "layer": 19, "node": "l:316", "routing": "mlp" }, { "cancel_layer": 26, "cancel_sublayer": "mlp", "layer": 20, "node": "l:296", "routing": "attn" }, { "cancel_layer": 19, "cancel_sublayer": "attn", "layer": 15, "node": "l:330", "routing": "mlp" }, { "cancel_layer": 19, "cancel_sublayer": "mlp", "layer": 18, "node": "l:337", "routing": "mlp" }, { "cancel_layer": 19, "cancel_sublayer": "mlp", "layer": 18, "node": "l:375", "routing": "mlp" }, { "cancel_layer": 26, "cancel_sublayer": "mlp", "layer": 19, "node": "l:328", "routing": "attn" }, { "cancel_layer": 18, "cancel_sublayer": "attn", "layer": 15, "node": "l:386", "routing": "mlp" }, { "cancel_layer": 18, "cancel_sublayer": "attn", "layer": 17, "node": "l:391", "routing": "mlp" }, { "cancel_layer": 18, "cancel_sublayer": "mlp", "layer": 17, "node": "l:396", "routing": "mlp" }, { "cancel_layer": 26, "cancel_sublayer": "mlp", "layer": 18, "node": "l:384", "routing": "attn" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 14, "node": "l:408", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 16, "node": "l:413", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "attn", "layer": 16, "node": "l:418", "routing": "mlp" }, { "cancel_layer": 26, "cancel_sublayer": "mlp", "layer": 17, "node": "l:406", "routing": "attn" }, { "cancel_layer": 5, "cancel_sublayer": "mlp", "layer": 4, "node": "l:436", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 4, "node": "l:439", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "mlp", "layer": 4, "node": "l:442", "routing": "mlp" }, { "cancel_layer": 26, "cancel_sublayer": "mlp", "layer": 5, "node": "l:434", "routing": "attn" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 4, "node": "l:457", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "mlp", "layer": 4, "node": "l:459", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "mlp", "layer": 4, "node": "l:461", "routing": "mlp" }, { "cancel_layer": 26, "cancel_sublayer": "mlp", "layer": 5, "node": "l:455", "routing": "attn" }, { "cancel_layer": 15, "cancel_sublayer": "mlp", "layer": 14, "node": "l:404", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "attn", "layer": 15, "node": "l:402", "routing": "mlp" }, { "cancel_layer": 22, "cancel_sublayer": "mlp", "layer": 21, "node": "l:31", "routing": "mlp" }, { "cancel_layer": 23, "cancel_sublayer": "attn", "layer": 22, "node": "l:272", "routing": "attn" }, { "cancel_layer": 21, "cancel_sublayer": "mlp", "layer": 20, "node": "l:274", "routing": "mlp" }, { "cancel_layer": 22, "cancel_sublayer": "mlp", "layer": 21, "node": "l:273", "routing": "mlp" }, { "cancel_layer": 21, "cancel_sublayer": "mlp", "layer": 20, "node": "l:295", "routing": "mlp" }, { "cancel_layer": 22, "cancel_sublayer": "mlp", "layer": 21, "node": "l:294", "routing": "mlp" }, { "cancel_layer": 20, "cancel_sublayer": "mlp", "layer": 19, "node": "l:327", "routing": "mlp" }, { "cancel_layer": 22, "cancel_sublayer": "mlp", "layer": 20, "node": "l:326", "routing": "mlp" }, { "cancel_layer": 22, "cancel_sublayer": "mlp", "layer": 21, "node": "l:383", "routing": "mlp" }, { "cancel_layer": 24, "cancel_sublayer": "attn", "layer": 22, "node": "l:270", "routing": "mlp" }, { "cancel_layer": 24, "cancel_sublayer": "attn", "layer": 22, "node": "l:292", "routing": "mlp" }, { "cancel_layer": 23, "cancel_sublayer": "mlp", "layer": 22, "node": "l:324", "routing": "mlp" }, { "cancel_layer": 24, "cancel_sublayer": "attn", "layer": 23, "node": "l:323", "routing": "mlp" }, { "cancel_layer": 23, "cancel_sublayer": "mlp", "layer": 22, "node": "l:381", "routing": "mlp" }, { "cancel_layer": 24, "cancel_sublayer": "mlp", "layer": 23, "node": "l:380", "routing": "mlp" }, { "cancel_layer": 25, "cancel_sublayer": "attn", "layer": 24, "node": "l:28", "routing": "attn" }, { "cancel_layer": 25, "cancel_sublayer": "attn", "layer": 16, "node": "l:401", "routing": "attn" }, { "cancel_layer": 26, "cancel_sublayer": "attn", "layer": 24, "node": "l:27", "routing": "mlp" }, { "cancel_layer": 26, "cancel_sublayer": "mlp", "layer": 25, "node": "l:423", "routing": "mlp" }, { "cancel_layer": 27, "cancel_sublayer": "mlp", "layer": 26, "node": "l:24", "routing": "mlp" }, { "cancel_layer": 28, "cancel_sublayer": "attn", "layer": 27, "node": "l:23", "routing": "attn" }, { "cancel_layer": 27, "cancel_sublayer": "mlp", "layer": 26, "node": "l:431", "routing": "mlp" }, { "cancel_layer": 28, "cancel_sublayer": "attn", "layer": 27, "node": "l:430", "routing": "attn" }, { "cancel_layer": 27, "cancel_sublayer": "mlp", "layer": 26, "node": "l:452", "routing": "mlp" }, { "cancel_layer": 28, "cancel_sublayer": "attn", "layer": 27, "node": "l:451", "routing": "attn" }, { "cancel_layer": 27, "cancel_sublayer": "mlp", "layer": 26, "node": "l:470", "routing": "mlp" }, { "cancel_layer": 28, "cancel_sublayer": "mlp", "layer": 27, "node": "l:469", "routing": "attn" }, { "cancel_layer": 27, "cancel_sublayer": "mlp", "layer": 26, "node": "l:480", "routing": "mlp" }, { "cancel_layer": 28, "cancel_sublayer": "attn", "layer": 27, "node": "l:479", "routing": "attn" }, { "cancel_layer": 27, "cancel_sublayer": "mlp", "layer": 26, "node": "l:490", "routing": "mlp" }, { "cancel_layer": 28, "cancel_sublayer": "attn", "layer": 27, "node": "l:489", "routing": "attn" }, { "cancel_layer": 27, "cancel_sublayer": "mlp", "layer": 26, "node": "l:500", "routing": "mlp" }, { "cancel_layer": 28, "cancel_sublayer": "attn", "layer": 27, "node": "l:499", "routing": "attn" }, { "cancel_layer": 27, "cancel_sublayer": "mlp", "layer": 26, "node": "l:510", "routing": "mlp" }, { "cancel_layer": 28, "cancel_sublayer": "attn", "layer": 27, "node": "l:509", "routing": "attn" }, { "cancel_layer": 1, "cancel_sublayer": "attn", "layer": 0, "node": "l:4", "routing": "mlp" }, { "cancel_layer": 1, "cancel_sublayer": "attn", "layer": 0, "node": "l:5", "routing": "mlp" }, { "cancel_layer": 1, "cancel_sublayer": "attn", "layer": 0, "node": "l:9", "routing": "mlp" }, { "cancel_layer": 29, "cancel_sublayer": "attn", "layer": 1, "node": "l:2", "routing": "attn" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:21", "routing": "mlp" }, { "cancel_layer": 2, "cancel_sublayer": "attn", "layer": 1, "node": "l:20", "routing": "mlp" }, { "cancel_layer": 3, "cancel_sublayer": "attn", "layer": 2, "node": "l:19", "routing": "attn" }, { "cancel_layer": 3, "cancel_sublayer": "mlp", "layer": 2, "node": "l:18", "routing": "mlp" }, { "cancel_layer": 4, "cancel_sublayer": "mlp", "layer": 3, "node": "l:17", "routing": "attn" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 3, "node": "l:16", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 3, "node": "l:22", "routing": "mlp" }, { "cancel_layer": 27, "cancel_sublayer": "mlp", "layer": 4, "node": "l:14", "routing": "mlp" }, { "cancel_layer": 28, "cancel_sublayer": "mlp", "layer": 27, "node": "l:12", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 3, "node": "l:428", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 3, "node": "l:429", "routing": "mlp" }, { "cancel_layer": 27, "cancel_sublayer": "mlp", "layer": 4, "node": "l:426", "routing": "mlp" }, { "cancel_layer": 28, "cancel_sublayer": "mlp", "layer": 27, "node": "l:424", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "mlp", "layer": 4, "node": "l:449", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "mlp", "layer": 4, "node": "l:450", "routing": "mlp" }, { "cancel_layer": 27, "cancel_sublayer": "mlp", "layer": 5, "node": "l:447", "routing": "mlp" }, { "cancel_layer": 28, "cancel_sublayer": "mlp", "layer": 27, "node": "l:445", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "mlp", "layer": 4, "node": "l:467", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 4, "node": "l:468", "routing": "mlp" }, { "cancel_layer": 28, "cancel_sublayer": "attn", "layer": 5, "node": "l:465", "routing": "mlp" }, { "cancel_layer": 28, "cancel_sublayer": "attn", "layer": 27, "node": "l:463", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "mlp", "layer": 4, "node": "l:477", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 4, "node": "l:478", "routing": "mlp" }, { "cancel_layer": 27, "cancel_sublayer": "mlp", "layer": 5, "node": "l:475", "routing": "mlp" }, { "cancel_layer": 28, "cancel_sublayer": "mlp", "layer": 27, "node": "l:473", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 3, "node": "l:487", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "attn", "layer": 3, "node": "l:488", "routing": "mlp" }, { "cancel_layer": 27, "cancel_sublayer": "mlp", "layer": 4, "node": "l:485", "routing": "mlp" }, { "cancel_layer": 28, "cancel_sublayer": "mlp", "layer": 27, "node": "l:483", "routing": "mlp" }, { "cancel_layer": 6, "cancel_sublayer": "attn", "layer": 4, "node": "l:497", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "mlp", "layer": 4, "node": "l:498", "routing": "mlp" }, { "cancel_layer": 27, "cancel_sublayer": "mlp", "layer": 5, "node": "l:495", "routing": "mlp" }, { "cancel_layer": 28, "cancel_sublayer": "mlp", "layer": 27, "node": "l:493", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "mlp", "layer": 4, "node": "l:507", "routing": "mlp" }, { "cancel_layer": 5, "cancel_sublayer": "mlp", "layer": 4, "node": "l:508", "routing": "mlp" }, { "cancel_layer": 27, "cancel_sublayer": "mlp", "layer": 5, "node": "l:505", "routing": "mlp" }, { "cancel_layer": 28, "cancel_sublayer": "mlp", "layer": 27, "node": "l:503", "routing": "mlp" }, { "cancel_layer": 29, "cancel_sublayer": "attn", "layer": 28, "node": "l:10", "routing": "attn" }, { "cancel_layer": 29, "cancel_sublayer": "attn", "layer": 27, "node": "l:513", "routing": "mlp" }, { "cancel_layer": 40, "cancel_sublayer": "attn", "layer": 28, "node": "l:0", "routing": "mlp" }, { "cancel_layer": 10, "cancel_sublayer": "attn", "layer": 7, "node": "l:229", "routing": "mlp" }, { "cancel_layer": 10, "cancel_sublayer": "attn", "layer": 7, "node": "l:191", "routing": "mlp" }, { "cancel_layer": 10, "cancel_sublayer": "attn", "layer": 7, "node": "l:142", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 4, "node": "l:221", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 4, "node": "l:201", "routing": "mlp" }, { "cancel_layer": 8, "cancel_sublayer": "attn", "layer": 4, "node": "l:90", "routing": "mlp" }, { "cancel_layer": 16, "cancel_sublayer": "mlp", "layer": 14, "node": "l:238", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "mlp", "layer": 14, "node": "l:257", "routing": "mlp" }, { "cancel_layer": 17, "cancel_sublayer": "mlp", "layer": 13, "node": "l:248", "routing": "mlp" }, { "cancel_layer": 18, "cancel_sublayer": "mlp", "layer": 13, "node": "l:265", "routing": "mlp" }, { "cancel_layer": 18, "cancel_sublayer": "mlp", "layer": 13, "node": "l:262", "routing": "mlp" }, { "cancel_layer": 18, "cancel_sublayer": "mlp", "layer": 13, "node": "l:269", "routing": "mlp" }, { "cancel_layer": 18, "cancel_sublayer": "mlp", "layer": 14, "node": "l:268", "routing": "mlp" }, { "cancel_layer": 19, "cancel_sublayer": "mlp", "layer": 16, "node": "l:179", "routing": "mlp" }, { "cancel_layer": 19, "cancel_sublayer": "mlp", "layer": 13, "node": "l:79", "routing": "mlp" }, { "cancel_layer": 24, "cancel_sublayer": "attn", "layer": 22, "node": "l:30", "routing": "mlp" }, { "cancel_layer": 26, "cancel_sublayer": "mlp", "layer": 25, "node": "l:26", "routing": "mlp" } ], "layers": [ { "active_attention_heads": 1, "active_mlp_neurons": 16, "attention_operations": [], "emitted_attention_heads": 0, "index": 0, "mlp_operations": [ { "hidden_slots": [ [ 1, 1 ] ], "id": "L0.mlp.0", "node": "l:46", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 17 ] ], "source_columns_b": null, "target_columns": [ [ 18, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 2, 1 ] ], "id": "L0.mlp.1", "node": "l:48", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 17 ] ], "source_columns_b": null, "target_columns": [ [ 19, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 3, 1 ] ], "id": "L0.mlp.2", "node": "l:50", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 17 ] ], "source_columns_b": null, "target_columns": [ [ 20, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 4, 1 ] ], "id": "L0.mlp.3", "node": "l:7", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 17 ] ], "source_columns_b": null, "target_columns": [ [ 21, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 5, 1 ] ], "id": "L0.mlp.4", "node": "l:106", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 17 ] ], "source_columns_b": null, "target_columns": [ [ 22, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 6, 10 ] ], "id": "L0.mlp.5", "node": "l:115", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 1, 17 ] ], "source_columns_b": null, "target_columns": [ [ 23, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [], "id": "L0.mlp.6", "node": "l:54", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 24, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.7", "node": "l:55", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 25, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.8", "node": "l:57", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 26, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.9", "node": "l:103", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 27, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.10", "node": "l:104", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 28, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.11", "node": "l:107", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 29, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.12", "node": "l:155", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 30, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.13", "node": "l:156", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 31, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.14", "node": "l:158", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 32, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.15", "node": "l:100", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 33, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.16", "node": "l:152", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 34, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.17", "node": "l:131", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 35, 4 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.18", "node": "l:4", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 39, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.19", "node": "l:5", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 40, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L0.mlp.20", "node": "l:9", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 41, 1 ] ], "type": "compute_literal_value" } ], "newly_computed": [ "l:46", "l:48", "l:50", "l:7", "l:54", "l:55", "l:57", "l:53", "l:56", "l:106", "l:115", "l:103", "l:104", "l:107", "l:102", "l:105", "l:100", "l:131", "l:130", "l:155", "l:156", "l:158", "l:154", "l:152", "l:4", "l:5", "l:9", "l:3", "l:6" ] }, { "active_attention_heads": 4, "active_mlp_neurons": 29, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L1.attn.0", "key_source_columns": [ [ 21, 1 ], [ 26, 1 ] ], "node": "l:52", "query_source_columns": [ [ 24, 2 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 21, 1 ] ], "source_columns_b": null, "target_columns": [ [ 24, 1 ] ], "type": "compute_attn" }, { "heads": [ 1, 1 ], "id": "L1.attn.1", "key_source_columns": [ [ 22, 1 ], [ 29, 1 ] ], "node": "l:101", "query_source_columns": [ [ 27, 2 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 22, 1 ] ], "source_columns_b": null, "target_columns": [ [ 25, 1 ] ], "type": "compute_attn" }, { "heads": [ 2, 1 ], "id": "L1.attn.2", "key_source_columns": [ [ 21, 1 ], [ 41, 1 ] ], "node": "l:2", "query_source_columns": [ [ 39, 2 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 21, 1 ] ], "source_columns_b": null, "target_columns": [ [ 26, 1 ] ], "type": "compute_attn" }, { "heads": [ 3, 1 ], "id": "L1.attn.3", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 24, 6 ], [ 39, 3 ] ], "type": "cancel" } ], "emitted_attention_heads": 4, "index": 1, "mlp_operations": [ { "hidden_slots": [ [ 1, 2 ] ], "id": "L1.mlp.0", "node": "l:45", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 18, 1 ] ], "source_columns_b": null, "target_columns": [ [ 27, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 3, 2 ] ], "id": "L1.mlp.1", "node": "l:47", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 19, 1 ] ], "source_columns_b": null, "target_columns": [ [ 28, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 5, 2 ] ], "id": "L1.mlp.2", "node": "l:49", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 20, 1 ] ], "source_columns_b": null, "target_columns": [ [ 29, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 7, 2 ] ], "id": "L1.mlp.3", "node": "l:51", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 24, 1 ] ], "source_columns_b": null, "target_columns": [ [ 40, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 9, 2 ] ], "id": "L1.mlp.4", "node": "l:166", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 24, 1 ] ], "source_columns_b": null, "target_columns": [ [ 39, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 11, 4 ] ], "id": "L1.mlp.5", "node": "l:129", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 22, 1 ], [ 35, 4 ] ], "source_columns_b": null, "target_columns": [ [ 42, 4 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 15, 2 ] ], "id": "L1.mlp.6", "node": "l:108", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 22, 1 ] ], "source_columns_b": null, "target_columns": [ [ 46, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 17, 2 ] ], "id": "L1.mlp.7", "node": "l:21", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 21, 1 ] ], "source_columns_b": null, "target_columns": [ [ 47, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [], "id": "L1.mlp.8", "node": "l:20", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 48, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [ [ 19, 8 ] ], "id": "L1.mlp.9", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 35, 4 ] ], "source_columns_b": null, "target_columns": [ [ 35, 4 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 27, 2 ] ], "id": "L1.mlp.10", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 22, 1 ] ], "source_columns_b": null, "target_columns": [ [ 22, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:45", "l:47", "l:49", "l:44", "l:52", "l:51", "l:166", "l:101", "l:108", "l:129", "l:2", "l:21", "l:20" ] }, { "active_attention_heads": 3, "active_mlp_neurons": 73, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L2.attn.0", "key_source_columns": [ [ 25, 1 ] ], "node": "l:99", "query_source_columns": [ [ 33, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 46, 1 ] ], "source_columns_b": null, "target_columns": [ [ 22, 1 ] ], "type": "compute_attn" }, { "heads": [ 1, 1 ], "id": "L2.attn.1", "key_source_columns": [ [ 26, 1 ] ], "node": "l:19", "query_source_columns": [ [ 48, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 47, 1 ] ], "source_columns_b": null, "target_columns": [ [ 24, 1 ] ], "type": "compute_attn" }, { "heads": [ 2, 1 ], "id": "L2.attn.2", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 24, 2 ], [ 33, 1 ], [ 46, 3 ] ], "type": "cancel" } ], "emitted_attention_heads": 3, "index": 2, "mlp_operations": [ { "hidden_slots": [ [ 1, 2 ] ], "id": "L2.mlp.0", "node": "l:43", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 27, 3 ] ], "source_columns_b": null, "target_columns": [ [ 25, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 3, 32 ] ], "id": "L2.mlp.1", "node": "l:98", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 22, 1 ] ], "source_columns_b": null, "target_columns": [ [ 33, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 35, 32 ] ], "id": "L2.mlp.2", "node": "l:18", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 24, 1 ] ], "source_columns_b": null, "target_columns": [ [ 35, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 67, 2 ] ], "id": "L2.mlp.3", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 27, 1 ] ], "source_columns_b": null, "target_columns": [ [ 27, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 69, 2 ] ], "id": "L2.mlp.4", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 28, 1 ] ], "source_columns_b": null, "target_columns": [ [ 28, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 71, 2 ] ], "id": "L2.mlp.5", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 29, 1 ] ], "source_columns_b": null, "target_columns": [ [ 29, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:43", "l:42", "l:99", "l:98", "l:19", "l:18" ] }, { "active_attention_heads": 3, "active_mlp_neurons": 25, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L3.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": "l:97", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 33, 1 ] ], "source_columns_b": null, "target_columns": [ [ 22, 1 ] ], "type": "compute_linear" }, { "heads": [ 1, 1 ], "id": "L3.attn.1", "key_source_columns": [ [ 0, 1 ] ], "node": "l:17", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 35, 1 ] ], "source_columns_b": null, "target_columns": [ [ 24, 1 ] ], "type": "compute_linear" }, { "heads": [ 2, 1 ], "id": "L3.attn.2", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 22, 1 ], [ 24, 1 ] ], "type": "cancel" } ], "emitted_attention_heads": 3, "index": 3, "mlp_operations": [ { "hidden_slots": [ [ 1, 2 ] ], "id": "L3.mlp.0", "node": "l:41", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 25, 1 ], [ 40, 1 ] ], "source_columns_b": null, "target_columns": [ [ 27, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 3, 2 ] ], "id": "L3.mlp.1", "node": "l:16", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 24, 1 ] ], "source_columns_b": null, "target_columns": [ [ 28, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 5, 2 ] ], "id": "L3.mlp.2", "node": "l:22", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 24, 1 ] ], "source_columns_b": null, "target_columns": [ [ 29, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 7, 2 ] ], "id": "L3.mlp.3", "node": "l:428", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 24, 1 ] ], "source_columns_b": null, "target_columns": [ [ 36, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 9, 2 ] ], "id": "L3.mlp.4", "node": "l:429", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 24, 1 ] ], "source_columns_b": null, "target_columns": [ [ 37, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 11, 2 ] ], "id": "L3.mlp.5", "node": "l:487", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 24, 1 ] ], "source_columns_b": null, "target_columns": [ [ 38, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 13, 2 ] ], "id": "L3.mlp.6", "node": "l:488", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 24, 1 ] ], "source_columns_b": null, "target_columns": [ [ 46, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 15, 2 ] ], "id": "L3.mlp.7", "node": "l:126", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 22, 1 ] ], "source_columns_b": null, "target_columns": [ [ 49, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [], "id": "L3.mlp.8", "node": "l:119", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 50, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L3.mlp.9", "node": "l:120", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 51, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L3.mlp.10", "node": "l:122", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 52, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L3.mlp.11", "node": "l:175", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 53, 4 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L3.mlp.12", "node": "l:93", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 57, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L3.mlp.13", "node": "l:94", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 58, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L3.mlp.14", "node": "l:96", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 59, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L3.mlp.15", "node": "l:38", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 60, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L3.mlp.16", "node": "l:39", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 61, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L3.mlp.17", "node": "l:58", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 62, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L3.mlp.18", "node": "l:64", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 63, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L3.mlp.19", "node": "l:65", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 64, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L3.mlp.20", "node": "l:67", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 65, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L3.mlp.21", "node": "l:73", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 66, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L3.mlp.22", "node": "l:74", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 67, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L3.mlp.23", "node": "l:76", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 68, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L3.mlp.24", "node": "l:59", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 69, 17 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L3.mlp.25", "node": "l:438", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 86, 17 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L3.mlp.26", "node": "l:441", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 103, 17 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L3.mlp.27", "node": "l:444", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 120, 17 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L3.mlp.28", "node": "l:97", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 22, 1 ] ], "type": "compute_bias" }, { "hidden_slots": [], "id": "L3.mlp.29", "node": "l:17", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 24, 1 ] ], "type": "compute_bias" }, { "hidden_slots": [ [ 17, 2 ] ], "id": "L3.mlp.30", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 25, 1 ] ], "source_columns_b": null, "target_columns": [ [ 25, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 19, 2 ] ], "id": "L3.mlp.31", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 40, 1 ] ], "source_columns_b": null, "target_columns": [ [ 40, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 21, 2 ] ], "id": "L3.mlp.32", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 33, 1 ] ], "source_columns_b": null, "target_columns": [ [ 33, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 23, 2 ] ], "id": "L3.mlp.33", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 35, 1 ] ], "source_columns_b": null, "target_columns": [ [ 35, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:41", "l:38", "l:39", "l:58", "l:37", "l:40", "l:64", "l:65", "l:67", "l:63", "l:66", "l:73", "l:74", "l:76", "l:72", "l:75", "l:119", "l:120", "l:122", "l:118", "l:121", "l:97", "l:126", "l:157", "l:175", "l:174", "l:93", "l:94", "l:96", "l:92", "l:95", "l:59", "l:438", "l:441", "l:444", "l:17", "l:16", "l:22", "l:15", "l:428", "l:429", "l:427", "l:487", "l:488", "l:486" ] }, { "active_attention_heads": 8, "active_mlp_neurons": 376, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L4.attn.0", "key_source_columns": [ [ 27, 1 ], [ 32, 1 ] ], "node": "l:153", "query_source_columns": [ [ 30, 2 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 27, 1 ] ], "source_columns_b": null, "target_columns": [ [ 30, 1 ] ], "type": "compute_attn" }, { "heads": [ 1, 1 ], "id": "L4.attn.1", "key_source_columns": [ [ 27, 1 ], [ 52, 1 ] ], "node": "l:117", "query_source_columns": [ [ 50, 2 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 27, 1 ] ], "source_columns_b": null, "target_columns": [ [ 31, 1 ] ], "type": "compute_attn" }, { "heads": [ 2, 1 ], "id": "L4.attn.2", "key_source_columns": [ [ 27, 1 ], [ 59, 1 ] ], "node": "l:91", "query_source_columns": [ [ 57, 2 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 22, 1 ] ], "source_columns_b": null, "target_columns": [ [ 32, 1 ] ], "type": "compute_attn" }, { "heads": [ 3, 1 ], "id": "L4.attn.3", "key_source_columns": [ [ 27, 1 ], [ 62, 1 ] ], "node": "l:36", "query_source_columns": [ [ 60, 2 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 18, 1 ] ], "source_columns_b": null, "target_columns": [ [ 33, 1 ] ], "type": "compute_attn" }, { "heads": [ 4, 1 ], "id": "L4.attn.4", "key_source_columns": [ [ 27, 1 ], [ 65, 1 ] ], "node": "l:62", "query_source_columns": [ [ 63, 2 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 19, 1 ] ], "source_columns_b": null, "target_columns": [ [ 35, 1 ] ], "type": "compute_attn" }, { "heads": [ 5, 1 ], "id": "L4.attn.5", "key_source_columns": [ [ 27, 1 ], [ 68, 1 ] ], "node": "l:71", "query_source_columns": [ [ 66, 2 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 20, 1 ] ], "source_columns_b": null, "target_columns": [ [ 40, 1 ] ], "type": "compute_attn" }, { "heads": [ 6, 1 ], "id": "L4.attn.6", "key_source_columns": [ [ 0, 1 ] ], "node": "l:127", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 49, 1 ] ], "source_columns_b": null, "target_columns": [ [ 50, 1 ] ], "type": "compute_linear" }, { "heads": [ 7, 1 ], "id": "L4.attn.7", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 18, 3 ], [ 22, 1 ], [ 30, 3 ], [ 50, 3 ], [ 57, 12 ] ], "type": "cancel" } ], "emitted_attention_heads": 8, "index": 4, "mlp_operations": [ { "hidden_slots": [ [ 1, 2 ] ], "id": "L4.mlp.0", "node": "l:116", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 31, 1 ] ], "source_columns_b": null, "target_columns": [ [ 51, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 3, 12 ] ], "id": "L4.mlp.1", "node": "l:124", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 49, 2 ] ], "source_columns_b": null, "target_columns": [ [ 52, 1 ], [ 57, 2 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 15, 2 ] ], "id": "L4.mlp.2", "node": "l:164", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 23, 1 ], [ 31, 1 ], [ 39, 1 ] ], "source_columns_b": null, "target_columns": [ [ 59, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 17, 4 ] ], "id": "L4.mlp.3", "node": "l:173", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 27, 1 ], [ 53, 4 ] ], "source_columns_b": null, "target_columns": [ [ 60, 4 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 21, 17 ] ], "id": "L4.mlp.4", "node": "l:221", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 32, 1 ] ], "source_columns_b": null, "target_columns": [ [ 64, 4 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 38, 13 ] ], "id": "L4.mlp.5", "node": "l:201", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 32, 1 ] ], "source_columns_b": null, "target_columns": [ [ 18, 3 ], [ 68, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 51, 9 ] ], "id": "L4.mlp.6", "node": "l:90", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 32, 1 ] ], "source_columns_b": null, "target_columns": [ [ 22, 1 ], [ 25, 1 ], [ 137, 2 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 60, 17 ] ], "id": "L4.mlp.7", "node": "l:34", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 33, 1 ], [ 69, 17 ] ], "source_columns_b": null, "target_columns": [ [ 139, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 77, 17 ] ], "id": "L4.mlp.8", "node": "l:277", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 33, 1 ], [ 69, 17 ] ], "source_columns_b": null, "target_columns": [ [ 156, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 94, 17 ] ], "id": "L4.mlp.9", "node": "l:436", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 33, 1 ], [ 86, 17 ] ], "source_columns_b": null, "target_columns": [ [ 173, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 111, 17 ] ], "id": "L4.mlp.10", "node": "l:439", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 35, 1 ], [ 103, 17 ] ], "source_columns_b": null, "target_columns": [ [ 190, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 128, 17 ] ], "id": "L4.mlp.11", "node": "l:442", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 40, 1 ], [ 120, 17 ] ], "source_columns_b": null, "target_columns": [ [ 207, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 145, 17 ] ], "id": "L4.mlp.12", "node": "l:457", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 33, 1 ], [ 86, 17 ] ], "source_columns_b": null, "target_columns": [ [ 224, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 162, 17 ] ], "id": "L4.mlp.13", "node": "l:459", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 35, 1 ], [ 103, 17 ] ], "source_columns_b": null, "target_columns": [ [ 241, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 179, 17 ] ], "id": "L4.mlp.14", "node": "l:461", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 40, 1 ], [ 120, 17 ] ], "source_columns_b": null, "target_columns": [ [ 258, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 196, 2 ] ], "id": "L4.mlp.15", "node": "l:449", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 24, 1 ] ], "source_columns_b": null, "target_columns": [ [ 275, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 198, 2 ] ], "id": "L4.mlp.16", "node": "l:450", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 24, 1 ] ], "source_columns_b": null, "target_columns": [ [ 276, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 200, 2 ] ], "id": "L4.mlp.17", "node": "l:467", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 24, 1 ] ], "source_columns_b": null, "target_columns": [ [ 277, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 202, 2 ] ], "id": "L4.mlp.18", "node": "l:468", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 24, 1 ] ], "source_columns_b": null, "target_columns": [ [ 278, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 204, 2 ] ], "id": "L4.mlp.19", "node": "l:477", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 24, 1 ] ], "source_columns_b": null, "target_columns": [ [ 279, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 206, 2 ] ], "id": "L4.mlp.20", "node": "l:478", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 24, 1 ] ], "source_columns_b": null, "target_columns": [ [ 280, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 208, 2 ] ], "id": "L4.mlp.21", "node": "l:497", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 24, 1 ] ], "source_columns_b": null, "target_columns": [ [ 281, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 210, 2 ] ], "id": "L4.mlp.22", "node": "l:498", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 24, 1 ] ], "source_columns_b": null, "target_columns": [ [ 282, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 212, 2 ] ], "id": "L4.mlp.23", "node": "l:507", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 24, 1 ] ], "source_columns_b": null, "target_columns": [ [ 283, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 214, 2 ] ], "id": "L4.mlp.24", "node": "l:508", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 24, 1 ] ], "source_columns_b": null, "target_columns": [ [ 284, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 216, 2 ] ], "id": "L4.mlp.25", "node": "l:14", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 28, 2 ] ], "source_columns_b": null, "target_columns": [ [ 285, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 218, 2 ] ], "id": "L4.mlp.26", "node": "l:426", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 36, 2 ] ], "source_columns_b": null, "target_columns": [ [ 286, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 220, 2 ] ], "id": "L4.mlp.27", "node": "l:485", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 38, 1 ], [ 46, 1 ] ], "source_columns_b": null, "target_columns": [ [ 287, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 222, 2 ] ], "id": "L4.mlp.28", "node": "l:159", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 27, 1 ] ], "source_columns_b": null, "target_columns": [ [ 288, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [], "id": "L4.mlp.29", "node": "l:178", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 289, 17 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L4.mlp.30", "node": "l:68", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 306, 17 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L4.mlp.31", "node": "l:127", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 50, 1 ] ], "type": "compute_bias" }, { "hidden_slots": [ [ 224, 34 ] ], "id": "L4.mlp.32", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 69, 17 ] ], "source_columns_b": null, "target_columns": [ [ 69, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 258, 34 ] ], "id": "L4.mlp.33", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 86, 17 ] ], "source_columns_b": null, "target_columns": [ [ 86, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 292, 34 ] ], "id": "L4.mlp.34", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 103, 17 ] ], "source_columns_b": null, "target_columns": [ [ 103, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 326, 34 ] ], "id": "L4.mlp.35", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 120, 17 ] ], "source_columns_b": null, "target_columns": [ [ 120, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 360, 8 ] ], "id": "L4.mlp.36", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 53, 4 ] ], "source_columns_b": null, "target_columns": [ [ 53, 4 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 368, 2 ] ], "id": "L4.mlp.37", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 27, 1 ] ], "source_columns_b": null, "target_columns": [ [ 27, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 370, 2 ] ], "id": "L4.mlp.38", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 39, 1 ] ], "source_columns_b": null, "target_columns": [ [ 39, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 372, 2 ] ], "id": "L4.mlp.39", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 49, 1 ] ], "source_columns_b": null, "target_columns": [ [ 49, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 374, 2 ] ], "id": "L4.mlp.40", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 24, 1 ] ], "source_columns_b": null, "target_columns": [ [ 24, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:36", "l:62", "l:71", "l:117", "l:116", "l:114", "l:165", "l:164", "l:127", "l:125", "l:124", "l:153", "l:159", "l:173", "l:178", "l:177", "l:91", "l:68", "l:35", "l:34", "l:61", "l:278", "l:277", "l:280", "l:315", "l:437", "l:436", "l:440", "l:439", "l:443", "l:442", "l:435", "l:458", "l:457", "l:460", "l:459", "l:462", "l:461", "l:456", "l:14", "l:426", "l:449", "l:450", "l:448", "l:467", "l:468", "l:466", "l:477", "l:478", "l:476", "l:485", "l:497", "l:498", "l:496", "l:507", "l:508", "l:506", "l:221", "l:201", "l:90" ] }, { "active_attention_heads": 4, "active_mlp_neurons": 364, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L5.attn.0", "key_source_columns": [ [ 30, 1 ] ], "node": "l:151", "query_source_columns": [ [ 34, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 288, 1 ] ], "source_columns_b": null, "target_columns": [ [ 24, 1 ] ], "type": "compute_attn" }, { "heads": [ 1, 1 ], "id": "L5.attn.1", "key_source_columns": [ [ 0, 1 ] ], "node": "l:434", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 173, 51 ] ], "source_columns_b": null, "target_columns": [ [ 27, 1 ], [ 69, 16 ] ], "type": "compute_linear" }, { "heads": [ 2, 1 ], "id": "L5.attn.2", "key_source_columns": [ [ 0, 1 ] ], "node": "l:455", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 224, 51 ] ], "source_columns_b": null, "target_columns": [ [ 85, 17 ] ], "type": "compute_linear" }, { "heads": [ 3, 1 ], "id": "L5.attn.3", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 28, 5 ], [ 34, 1 ], [ 36, 3 ], [ 46, 1 ], [ 50, 1 ], [ 190, 17 ], [ 224, 17 ], [ 288, 1 ] ], "type": "cancel" } ], "emitted_attention_heads": 4, "index": 5, "mlp_operations": [ { "hidden_slots": [ [ 1, 32 ] ], "id": "L5.mlp.0", "node": "l:150", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 24, 1 ] ], "source_columns_b": null, "target_columns": [ [ 102, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 33, 2 ] ], "id": "L5.mlp.1", "node": "l:113", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 23, 1 ], [ 51, 1 ] ], "source_columns_b": null, "target_columns": [ [ 103, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 35, 34 ] ], "id": "L5.mlp.2", "node": "l:176", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 59, 1 ], [ 1, 17 ], [ 289, 17 ] ], "source_columns_b": null, "target_columns": [ [ 104, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 69, 17 ] ], "id": "L5.mlp.3", "node": "l:60", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 35, 1 ], [ 306, 17 ] ], "source_columns_b": null, "target_columns": [ [ 28, 1 ], [ 121, 16 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 86, 17 ] ], "id": "L5.mlp.4", "node": "l:279", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 35, 1 ], [ 306, 17 ] ], "source_columns_b": null, "target_columns": [ [ 29, 4 ], [ 34, 1 ], [ 36, 4 ], [ 190, 8 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 103, 17 ] ], "id": "L5.mlp.5", "node": "l:314", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 35, 1 ], [ 306, 17 ] ], "source_columns_b": null, "target_columns": [ [ 46, 1 ], [ 198, 9 ], [ 224, 7 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 120, 2 ] ], "id": "L5.mlp.6", "node": "l:447", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 275, 2 ] ], "source_columns_b": null, "target_columns": [ [ 231, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 122, 2 ] ], "id": "L5.mlp.7", "node": "l:465", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 277, 2 ] ], "source_columns_b": null, "target_columns": [ [ 232, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 124, 2 ] ], "id": "L5.mlp.8", "node": "l:475", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 279, 2 ] ], "source_columns_b": null, "target_columns": [ [ 233, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 126, 2 ] ], "id": "L5.mlp.9", "node": "l:495", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 281, 2 ] ], "source_columns_b": null, "target_columns": [ [ 234, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 128, 2 ] ], "id": "L5.mlp.10", "node": "l:505", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 283, 2 ] ], "source_columns_b": null, "target_columns": [ [ 235, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 130, 6 ] ], "id": "L5.mlp.11", "node": "l:123", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 52, 1 ], [ 57, 2 ] ], "source_columns_b": null, "target_columns": [ [ 236, 3 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [], "id": "L5.mlp.12", "node": "l:134", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 49, 2 ], [ 53, 4 ], [ 239, 2 ], [ 288, 1 ], [ 323, 8 ] ], "type": "compute_literal_value" }, { "hidden_slots": [ [ 136, 34 ] ], "id": "L5.mlp.13", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 289, 17 ] ], "source_columns_b": null, "target_columns": [ [ 289, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 170, 34 ] ], "id": "L5.mlp.14", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 306, 17 ] ], "source_columns_b": null, "target_columns": [ [ 306, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 204, 34 ] ], "id": "L5.mlp.15", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 173, 17 ] ], "source_columns_b": null, "target_columns": [ [ 173, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 238, 34 ] ], "id": "L5.mlp.16", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 207, 17 ] ], "source_columns_b": null, "target_columns": [ [ 207, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 272, 34 ] ], "id": "L5.mlp.17", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 241, 17 ] ], "source_columns_b": null, "target_columns": [ [ 241, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 306, 34 ] ], "id": "L5.mlp.18", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 258, 17 ] ], "source_columns_b": null, "target_columns": [ [ 258, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 340, 6 ] ], "id": "L5.mlp.19", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 52, 1 ], [ 57, 2 ] ], "source_columns_b": null, "target_columns": [ [ 52, 1 ], [ 57, 2 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 346, 2 ] ], "id": "L5.mlp.20", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 23, 1 ] ], "source_columns_b": null, "target_columns": [ [ 23, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 348, 2 ] ], "id": "L5.mlp.21", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 51, 1 ] ], "source_columns_b": null, "target_columns": [ [ 51, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 350, 2 ] ], "id": "L5.mlp.22", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 275, 1 ] ], "source_columns_b": null, "target_columns": [ [ 275, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 352, 2 ] ], "id": "L5.mlp.23", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 276, 1 ] ], "source_columns_b": null, "target_columns": [ [ 276, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 354, 2 ] ], "id": "L5.mlp.24", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 277, 1 ] ], "source_columns_b": null, "target_columns": [ [ 277, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 356, 2 ] ], "id": "L5.mlp.25", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 279, 1 ] ], "source_columns_b": null, "target_columns": [ [ 279, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 358, 2 ] ], "id": "L5.mlp.26", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 282, 1 ] ], "source_columns_b": null, "target_columns": [ [ 282, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 360, 2 ] ], "id": "L5.mlp.27", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 283, 1 ] ], "source_columns_b": null, "target_columns": [ [ 283, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 362, 2 ] ], "id": "L5.mlp.28", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 284, 1 ] ], "source_columns_b": null, "target_columns": [ [ 284, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:113", "l:123", "l:112", "l:134", "l:133", "l:151", "l:150", "l:176", "l:60", "l:279", "l:314", "l:434", "l:455", "l:512", "l:447", "l:465", "l:475", "l:495", "l:505" ] }, { "active_attention_heads": 2, "active_mlp_neurons": 82, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L6.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": "l:149", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 102, 1 ] ], "source_columns_b": null, "target_columns": [ [ 51, 1 ] ], "type": "compute_linear" }, { "heads": [ 1, 1 ], "id": "L6.attn.1", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 24, 1 ], [ 102, 1 ], [ 278, 1 ], [ 280, 2 ] ], "type": "cancel" } ], "emitted_attention_heads": 2, "index": 6, "mlp_operations": [ { "hidden_slots": [ [ 1, 3 ] ], "id": "L6.mlp.0", "node": "l:111", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 103, 1 ], [ 236, 3 ] ], "source_columns_b": null, "target_columns": [ [ 23, 1 ], [ 52, 1 ], [ 102, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 4, 34 ] ], "id": "L6.mlp.1", "node": "l:132", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 103, 1 ], [ 1, 17 ], [ 49, 2 ], [ 53, 4 ], [ 239, 2 ], [ 288, 1 ], [ 323, 8 ] ], "source_columns_b": null, "target_columns": [ [ 24, 1 ], [ 173, 16 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 38, 2 ] ], "id": "L6.mlp.2", "node": "l:170", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 51, 1 ] ], "source_columns_b": null, "target_columns": [ [ 189, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [], "id": "L6.mlp.3", "node": "l:145", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 207, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L6.mlp.4", "node": "l:146", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 208, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L6.mlp.5", "node": "l:148", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 209, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L6.mlp.6", "node": "l:128", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 210, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L6.mlp.7", "node": "l:149", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 51, 1 ] ], "type": "compute_bias" }, { "hidden_slots": [ [ 40, 34 ] ], "id": "L6.mlp.8", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 49, 2 ], [ 53, 4 ], [ 239, 2 ], [ 288, 1 ], [ 323, 8 ] ], "source_columns_b": null, "target_columns": [ [ 49, 2 ], [ 53, 4 ], [ 239, 2 ], [ 288, 1 ], [ 323, 8 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 74, 6 ] ], "id": "L6.mlp.9", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 236, 3 ] ], "source_columns_b": null, "target_columns": [ [ 236, 3 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 80, 2 ] ], "id": "L6.mlp.10", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 103, 1 ] ], "source_columns_b": null, "target_columns": [ [ 103, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:111", "l:128", "l:110", "l:132", "l:149", "l:170", "l:145", "l:146", "l:148", "l:144", "l:147" ] }, { "active_attention_heads": 4, "active_mlp_neurons": 54, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L7.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": "l:109", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": 1, "role": "transport", "source_columns": [ [ 23, 1 ], [ 52, 1 ], [ 102, 1 ], [ 210, 1 ] ], "source_columns_b": null, "target_columns": [ [ 42, 4 ] ], "type": "add_into" }, { "heads": [ 1, 1 ], "id": "L7.attn.1", "key_source_columns": [ [ 21, 1 ], [ 209, 1 ] ], "node": "l:143", "query_source_columns": [ [ 207, 2 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 51, 1 ] ], "source_columns_b": null, "target_columns": [ [ 326, 1 ] ], "type": "compute_attn" }, { "heads": [ 2, 1 ], "id": "L7.attn.2", "key_source_columns": [ [ 0, 1 ] ], "node": "l:171", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 189, 1 ] ], "source_columns_b": null, "target_columns": [ [ 50, 1 ] ], "type": "compute_linear" }, { "heads": [ 3, 1 ], "id": "L7.attn.3", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 1, 17 ], [ 51, 1 ], [ 207, 3 ] ], "type": "cancel" } ], "emitted_attention_heads": 4, "index": 7, "mlp_operations": [ { "hidden_slots": [ [ 1, 12 ] ], "id": "L7.mlp.0", "node": "l:168", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 189, 1 ], [ 50, 1 ] ], "source_columns_b": null, "target_columns": [ [ 51, 1 ], [ 53, 2 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 13, 17 ] ], "id": "L7.mlp.1", "node": "l:229", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 326, 1 ] ], "source_columns_b": null, "target_columns": [ [ 55, 2 ], [ 328, 2 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 30, 13 ] ], "id": "L7.mlp.2", "node": "l:191", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 326, 1 ] ], "source_columns_b": null, "target_columns": [ [ 103, 1 ], [ 207, 1 ], [ 327, 1 ], [ 330, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 43, 9 ] ], "id": "L7.mlp.3", "node": "l:142", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 326, 1 ] ], "source_columns_b": null, "target_columns": [ [ 208, 2 ], [ 211, 2 ] ], "type": "compute_ffn" }, { "hidden_slots": [], "id": "L7.mlp.4", "node": "l:171", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 50, 1 ] ], "type": "compute_bias" }, { "hidden_slots": [ [ 52, 2 ] ], "id": "L7.mlp.5", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 189, 1 ] ], "source_columns_b": null, "target_columns": [ [ 189, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:109", "l:171", "l:169", "l:168", "l:143", "l:229", "l:191", "l:142" ] }, { "active_attention_heads": 5, "active_mlp_neurons": 6, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L8.attn.0", "key_source_columns": [ [ 42, 4 ] ], "node": "l:220", "query_source_columns": [ [ 64, 4 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 24, 1 ], [ 173, 16 ] ], "source_columns_b": null, "target_columns": [ [ 18, 3 ], [ 22, 2 ], [ 44, 1 ], [ 50, 5 ], [ 64, 2 ], [ 67, 2 ], [ 102, 1 ], [ 326, 1 ] ], "type": "compute_attn" }, { "heads": [ 1, 1 ], "id": "L8.attn.1", "key_source_columns": [ [ 42, 4 ] ], "node": "l:89", "query_source_columns": [ [ 22, 1 ], [ 25, 1 ], [ 137, 2 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 24, 1 ], [ 173, 16 ] ], "source_columns_b": null, "target_columns": [ [ 24, 2 ], [ 66, 1 ], [ 137, 2 ], [ 173, 12 ] ], "type": "compute_attn" }, { "heads": [ 2, 1 ], "id": "L8.attn.2", "key_source_columns": [ [ 42, 4 ] ], "node": "l:200", "query_source_columns": [ [ 18, 3 ], [ 68, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 24, 1 ], [ 173, 16 ] ], "source_columns_b": null, "target_columns": [ [ 42, 2 ], [ 185, 5 ], [ 210, 1 ], [ 213, 9 ] ], "type": "compute_attn" }, { "heads": [ 3, 1 ], "id": "L8.attn.3", "key_source_columns": [ [ 0, 1 ] ], "node": "l:167", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 51, 1 ], [ 53, 2 ] ], "source_columns_b": null, "target_columns": [ [ 45, 1 ], [ 222, 2 ] ], "type": "compute_linear" }, { "heads": [ 4, 1 ], "id": "L8.attn.4", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 18, 3 ], [ 22, 4 ], [ 42, 4 ], [ 50, 5 ], [ 64, 5 ], [ 102, 1 ], [ 137, 2 ], [ 173, 16 ], [ 210, 1 ], [ 326, 1 ] ], "type": "cancel" } ], "emitted_attention_heads": 5, "index": 8, "mlp_operations": [ { "hidden_slots": [ [ 1, 3 ] ], "id": "L8.mlp.0", "node": "l:162", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 59, 1 ], [ 45, 1 ], [ 222, 2 ] ], "source_columns_b": null, "target_columns": [ [ 236, 3 ] ], "type": "compute_ffn" }, { "hidden_slots": [], "id": "L8.mlp.1", "node": "l:172", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 49, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L8.mlp.2", "node": "l:86", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 239, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L8.mlp.3", "node": "l:167", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 45, 1 ], [ 222, 2 ] ], "type": "compute_bias" }, { "hidden_slots": [ [ 4, 2 ] ], "id": "L8.mlp.4", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 59, 1 ] ], "source_columns_b": null, "target_columns": [ [ 59, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:167", "l:163", "l:162", "l:172", "l:161", "l:89", "l:86", "l:200", "l:220" ] }, { "active_attention_heads": 2, "active_mlp_neurons": 1, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L9.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": "l:160", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": 1, "role": "transport", "source_columns": [ [ 236, 3 ], [ 49, 1 ] ], "source_columns_b": null, "target_columns": [ [ 60, 4 ] ], "type": "add_into" }, { "heads": [ 1, 1 ], "id": "L9.attn.1", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 45, 1 ], [ 222, 2 ] ], "type": "cancel" } ], "emitted_attention_heads": 2, "index": 9, "mlp_operations": [], "newly_computed": [ "l:160" ] }, { "active_attention_heads": 4, "active_mlp_neurons": 3, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L10.attn.0", "key_source_columns": [ [ 60, 4 ] ], "node": "l:228", "query_source_columns": [ [ 55, 2 ], [ 328, 2 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 104, 17 ] ], "source_columns_b": null, "target_columns": [ [ 55, 2 ], [ 59, 5 ], [ 103, 8 ], [ 328, 2 ] ], "type": "compute_attn" }, { "heads": [ 1, 1 ], "id": "L10.attn.1", "key_source_columns": [ [ 60, 4 ] ], "node": "l:141", "query_source_columns": [ [ 208, 2 ], [ 211, 2 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 104, 17 ] ], "source_columns_b": null, "target_columns": [ [ 111, 10 ], [ 207, 3 ], [ 211, 2 ], [ 222, 1 ], [ 327, 1 ] ], "type": "compute_attn" }, { "heads": [ 2, 1 ], "id": "L10.attn.2", "key_source_columns": [ [ 60, 4 ] ], "node": "l:190", "query_source_columns": [ [ 103, 1 ], [ 207, 1 ], [ 327, 1 ], [ 330, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 104, 17 ] ], "source_columns_b": null, "target_columns": [ [ 45, 1 ], [ 223, 1 ], [ 236, 3 ], [ 240, 12 ] ], "type": "compute_attn" }, { "heads": [ 3, 1 ], "id": "L10.attn.3", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 55, 2 ], [ 60, 4 ], [ 103, 18 ], [ 207, 3 ], [ 211, 2 ], [ 236, 3 ], [ 327, 4 ] ], "type": "cancel" } ], "emitted_attention_heads": 4, "index": 10, "mlp_operations": [ { "hidden_slots": [], "id": "L10.mlp.0", "node": "l:216", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 252, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L10.mlp.1", "node": "l:217", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 253, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L10.mlp.2", "node": "l:219", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 254, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L10.mlp.3", "node": "l:224", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 255, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L10.mlp.4", "node": "l:225", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 256, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L10.mlp.5", "node": "l:227", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 257, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L10.mlp.6", "node": "l:85", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 258, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L10.mlp.7", "node": "l:88", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 259, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L10.mlp.8", "node": "l:196", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 260, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L10.mlp.9", "node": "l:197", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 261, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L10.mlp.10", "node": "l:199", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 262, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L10.mlp.11", "node": "l:137", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 263, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L10.mlp.12", "node": "l:138", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 264, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L10.mlp.13", "node": "l:140", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 265, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L10.mlp.14", "node": "l:186", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 266, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L10.mlp.15", "node": "l:187", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 267, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L10.mlp.16", "node": "l:189", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 268, 1 ] ], "type": "compute_literal_value" }, { "hidden_slots": [ [ 1, 2 ] ], "id": "L10.mlp.17", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 49, 1 ] ], "source_columns_b": null, "target_columns": [ [ 49, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:85", "l:88", "l:84", "l:87", "l:196", "l:197", "l:199", "l:195", "l:198", "l:216", "l:217", "l:219", "l:215", "l:218", "l:141", "l:137", "l:138", "l:140", "l:136", "l:139", "l:190", "l:186", "l:187", "l:189", "l:185", "l:188", "l:228", "l:224", "l:225", "l:227", "l:223", "l:226" ] }, { "active_attention_heads": 8, "active_mlp_neurons": 1543, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L11.attn.0", "key_source_columns": [ [ 21, 1 ], [ 254, 1 ] ], "node": "l:214", "query_source_columns": [ [ 252, 2 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 18, 3 ], [ 22, 2 ], [ 44, 1 ], [ 50, 5 ], [ 64, 2 ], [ 67, 2 ], [ 102, 1 ], [ 326, 1 ] ], "source_columns_b": null, "target_columns": [ [ 18, 3 ], [ 55, 2 ], [ 59, 7 ], [ 68, 1 ], [ 106, 1 ], [ 326, 1 ], [ 328, 2 ] ], "type": "compute_attn" }, { "heads": [ 1, 1 ], "id": "L11.attn.1", "key_source_columns": [ [ 21, 1 ], [ 257, 1 ] ], "node": "l:222", "query_source_columns": [ [ 255, 2 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 55, 2 ], [ 59, 5 ], [ 103, 8 ], [ 328, 2 ] ], "source_columns_b": null, "target_columns": [ [ 21, 5 ], [ 66, 1 ], [ 102, 4 ], [ 107, 4 ], [ 137, 2 ], [ 173, 1 ] ], "type": "compute_attn" }, { "heads": [ 2, 1 ], "id": "L11.attn.2", "key_source_columns": [ [ 21, 1 ], [ 259, 1 ] ], "node": "l:83", "query_source_columns": [ [ 258, 1 ], [ 239, 1 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 24, 2 ], [ 66, 1 ], [ 137, 2 ], [ 173, 12 ] ], "source_columns_b": null, "target_columns": [ [ 174, 16 ], [ 210, 1 ] ], "type": "compute_attn" }, { "heads": [ 3, 1 ], "id": "L11.attn.3", "key_source_columns": [ [ 21, 1 ], [ 262, 1 ] ], "node": "l:194", "query_source_columns": [ [ 260, 2 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 42, 2 ], [ 185, 5 ], [ 210, 1 ], [ 213, 9 ] ], "source_columns_b": null, "target_columns": [ [ 42, 4 ], [ 213, 9 ], [ 223, 1 ], [ 236, 3 ] ], "type": "compute_attn" }, { "heads": [ 4, 1 ], "id": "L11.attn.4", "key_source_columns": [ [ 21, 1 ], [ 265, 1 ] ], "node": "l:135", "query_source_columns": [ [ 263, 2 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 111, 10 ], [ 207, 3 ], [ 211, 2 ], [ 222, 1 ], [ 327, 1 ] ], "source_columns_b": null, "target_columns": [ [ 49, 2 ], [ 239, 8 ], [ 248, 3 ], [ 252, 4 ] ], "type": "compute_attn" }, { "heads": [ 5, 1 ], "id": "L11.attn.5", "key_source_columns": [ [ 21, 1 ], [ 268, 1 ] ], "node": "l:184", "query_source_columns": [ [ 266, 2 ] ], "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 45, 1 ], [ 223, 1 ], [ 236, 3 ], [ 240, 12 ] ], "source_columns_b": null, "target_columns": [ [ 51, 4 ], [ 67, 1 ], [ 258, 5 ], [ 266, 7 ] ], "type": "compute_attn" }, { "heads": [ 6, 2 ], "id": "L11.attn.6", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 18, 8 ], [ 42, 4 ], [ 50, 7 ], [ 59, 10 ], [ 102, 9 ], [ 137, 2 ], [ 173, 17 ], [ 210, 1 ], [ 213, 9 ], [ 223, 1 ], [ 236, 20 ], [ 257, 6 ], [ 266, 3 ], [ 326, 1 ], [ 328, 2 ] ], "type": "cancel" } ], "emitted_attention_heads": 8, "index": 11, "mlp_operations": [ { "hidden_slots": [ [ 1, 100 ] ], "id": "L11.mlp.0", "node": "l:212", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 18, 3 ], [ 55, 2 ], [ 59, 7 ], [ 68, 1 ], [ 106, 1 ], [ 326, 1 ], [ 328, 2 ], [ 21, 5 ], [ 66, 1 ], [ 102, 4 ], [ 107, 4 ], [ 137, 2 ], [ 173, 1 ] ], "source_columns_b": null, "target_columns": [ [ 57, 2 ], [ 273, 12 ], [ 288, 20 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 101, 100 ] ], "id": "L11.mlp.1", "node": "l:81", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 174, 16 ], [ 210, 1 ], [ 49, 2 ], [ 239, 8 ], [ 248, 3 ], [ 252, 4 ] ], "source_columns_b": null, "target_columns": [ [ 308, 11 ], [ 320, 3 ], [ 331, 20 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 201, 100 ] ], "id": "L11.mlp.2", "node": "l:182", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 174, 16 ], [ 210, 1 ], [ 51, 4 ], [ 67, 1 ], [ 258, 5 ], [ 266, 7 ] ], "source_columns_b": null, "target_columns": [ [ 351, 34 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 301, 100 ] ], "id": "L11.mlp.3", "node": "l:251", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 174, 16 ], [ 210, 1 ], [ 21, 5 ], [ 66, 1 ], [ 102, 4 ], [ 107, 4 ], [ 137, 2 ], [ 173, 1 ] ], "source_columns_b": null, "target_columns": [ [ 385, 34 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 401, 100 ] ], "id": "L11.mlp.4", "node": "l:192", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 42, 4 ], [ 213, 9 ], [ 223, 1 ], [ 236, 3 ], [ 49, 2 ], [ 239, 8 ], [ 248, 3 ], [ 252, 4 ] ], "source_columns_b": null, "target_columns": [ [ 419, 34 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 501, 100 ] ], "id": "L11.mlp.5", "node": "l:253", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 42, 4 ], [ 213, 9 ], [ 223, 1 ], [ 236, 3 ], [ 51, 4 ], [ 67, 1 ], [ 258, 5 ], [ 266, 7 ] ], "source_columns_b": null, "target_columns": [ [ 453, 34 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 601, 100 ] ], "id": "L11.mlp.6", "node": "l:241", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 42, 4 ], [ 213, 9 ], [ 223, 1 ], [ 236, 3 ], [ 21, 5 ], [ 66, 1 ], [ 102, 4 ], [ 107, 4 ], [ 137, 2 ], [ 173, 1 ] ], "source_columns_b": null, "target_columns": [ [ 487, 34 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 701, 100 ] ], "id": "L11.mlp.7", "node": "l:255", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 18, 3 ], [ 55, 2 ], [ 59, 7 ], [ 68, 1 ], [ 106, 1 ], [ 326, 1 ], [ 328, 2 ], [ 49, 2 ], [ 239, 8 ], [ 248, 3 ], [ 252, 4 ] ], "source_columns_b": null, "target_columns": [ [ 521, 34 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 801, 100 ] ], "id": "L11.mlp.8", "node": "l:243", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 18, 3 ], [ 55, 2 ], [ 59, 7 ], [ 68, 1 ], [ 106, 1 ], [ 326, 1 ], [ 328, 2 ], [ 51, 4 ], [ 67, 1 ], [ 258, 5 ], [ 266, 7 ] ], "source_columns_b": null, "target_columns": [ [ 555, 34 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 901, 100 ] ], "id": "L11.mlp.9", "node": "l:347", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 174, 16 ], [ 210, 1 ], [ 49, 2 ], [ 239, 8 ], [ 248, 3 ], [ 252, 4 ] ], "source_columns_b": null, "target_columns": [ [ 589, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 1001, 100 ] ], "id": "L11.mlp.10", "node": "l:351", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 42, 4 ], [ 213, 9 ], [ 223, 1 ], [ 236, 3 ], [ 51, 4 ], [ 67, 1 ], [ 258, 5 ], [ 266, 7 ] ], "source_columns_b": null, "target_columns": [ [ 590, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 1101, 100 ] ], "id": "L11.mlp.11", "node": "l:355", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 18, 3 ], [ 55, 2 ], [ 59, 7 ], [ 68, 1 ], [ 106, 1 ], [ 326, 1 ], [ 328, 2 ], [ 21, 5 ], [ 66, 1 ], [ 102, 4 ], [ 107, 4 ], [ 137, 2 ], [ 173, 1 ] ], "source_columns_b": null, "target_columns": [ [ 591, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 1201, 100 ] ], "id": "L11.mlp.12", "node": "l:310", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 42, 4 ], [ 213, 9 ], [ 223, 1 ], [ 236, 3 ], [ 51, 4 ], [ 67, 1 ], [ 258, 5 ], [ 266, 7 ] ], "source_columns_b": null, "target_columns": [ [ 592, 3 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 1301, 100 ] ], "id": "L11.mlp.13", "node": "l:312", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 174, 16 ], [ 210, 1 ], [ 49, 2 ], [ 239, 8 ], [ 248, 3 ], [ 252, 4 ] ], "source_columns_b": null, "target_columns": [ [ 595, 3 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 1401, 100 ] ], "id": "L11.mlp.14", "node": "l:306", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 18, 3 ], [ 55, 2 ], [ 59, 7 ], [ 68, 1 ], [ 106, 1 ], [ 326, 1 ], [ 328, 2 ], [ 21, 5 ], [ 66, 1 ], [ 102, 4 ], [ 107, 4 ], [ 137, 2 ], [ 173, 1 ] ], "source_columns_b": null, "target_columns": [ [ 598, 3 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 1501, 34 ] ], "id": "L11.mlp.15", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 111, 10 ], [ 207, 3 ], [ 211, 2 ], [ 222, 1 ], [ 327, 1 ] ], "source_columns_b": null, "target_columns": [ [ 111, 10 ], [ 207, 3 ], [ 211, 2 ], [ 222, 1 ], [ 327, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1535, 2 ] ], "id": "L11.mlp.16", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 263, 1 ] ], "source_columns_b": null, "target_columns": [ [ 263, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1537, 2 ] ], "id": "L11.mlp.17", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 264, 1 ] ], "source_columns_b": null, "target_columns": [ [ 264, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1539, 2 ] ], "id": "L11.mlp.18", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 265, 1 ] ], "source_columns_b": null, "target_columns": [ [ 265, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1541, 2 ] ], "id": "L11.mlp.19", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 256, 1 ] ], "source_columns_b": null, "target_columns": [ [ 256, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:83", "l:194", "l:214", "l:135", "l:184", "l:222", "l:307", "l:306", "l:311", "l:310", "l:313", "l:312", "l:336", "l:309", "l:348", "l:347", "l:352", "l:351", "l:356", "l:355", "l:82", "l:81", "l:183", "l:182", "l:252", "l:251", "l:193", "l:192", "l:254", "l:253", "l:242", "l:241", "l:256", "l:255", "l:244", "l:243", "l:213", "l:212", "l:240", "l:250", "l:264", "l:181" ] }, { "active_attention_heads": 12, "active_mlp_neurons": 399, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L12.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": "l:211", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 57, 2 ], [ 273, 12 ], [ 288, 20 ] ], "source_columns_b": null, "target_columns": [ [ 120, 1 ] ], "type": "compute_linear" }, { "heads": [ 1, 3 ], "id": "L12.attn.1", "key_source_columns": [ [ 0, 1 ] ], "node": "l:249", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 385, 34 ], [ 453, 136 ] ], "source_columns_b": null, "target_columns": [ [ 327, 1 ] ], "type": "compute_linear" }, { "heads": [ 4, 3 ], "id": "L12.attn.2", "key_source_columns": [ [ 0, 1 ] ], "node": "l:263", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 351, 136 ], [ 521, 34 ] ], "source_columns_b": null, "target_columns": [ [ 207, 1 ] ], "type": "compute_linear" }, { "heads": [ 7, 2 ], "id": "L12.attn.3", "key_source_columns": [ [ 0, 1 ] ], "node": "l:180", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 308, 11 ], [ 320, 3 ], [ 331, 54 ], [ 419, 34 ] ], "source_columns_b": null, "target_columns": [ [ 208, 1 ] ], "type": "compute_linear" }, { "heads": [ 9, 3 ], "id": "L12.attn.4", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 351, 34 ], [ 419, 68 ], [ 521, 34 ] ], "type": "cancel" } ], "emitted_attention_heads": 12, "index": 12, "mlp_operations": [ { "hidden_slots": [ [ 1, 2 ] ], "id": "L12.mlp.0", "node": "l:346", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 589, 1 ] ], "source_columns_b": null, "target_columns": [ [ 209, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 3, 2 ] ], "id": "L12.mlp.1", "node": "l:349", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 589, 1 ] ], "source_columns_b": null, "target_columns": [ [ 211, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 5, 2 ] ], "id": "L12.mlp.2", "node": "l:350", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 590, 1 ] ], "source_columns_b": null, "target_columns": [ [ 212, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 7, 2 ] ], "id": "L12.mlp.3", "node": "l:353", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 590, 1 ] ], "source_columns_b": null, "target_columns": [ [ 222, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 9, 2 ] ], "id": "L12.mlp.4", "node": "l:354", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 591, 1 ] ], "source_columns_b": null, "target_columns": [ [ 256, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 11, 2 ] ], "id": "L12.mlp.5", "node": "l:357", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 591, 1 ] ], "source_columns_b": null, "target_columns": [ [ 263, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 13, 9 ] ], "id": "L12.mlp.6", "node": "l:308", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 592, 6 ] ], "source_columns_b": null, "target_columns": [ [ 264, 2 ], [ 351, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 22, 9 ] ], "id": "L12.mlp.7", "node": "l:335", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 598, 3 ], [ 592, 3 ] ], "source_columns_b": null, "target_columns": [ [ 352, 3 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 31, 2 ] ], "id": "L12.mlp.8", "node": "l:230", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 120, 1 ] ], "source_columns_b": null, "target_columns": [ [ 355, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 33, 2 ] ], "id": "L12.mlp.9", "node": "l:239", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 487, 34 ], [ 555, 34 ], [ 57, 2 ], [ 273, 12 ], [ 288, 20 ] ], "source_columns_b": null, "target_columns": [ [ 356, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 35, 2 ] ], "id": "L12.mlp.10", "node": "l:80", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 308, 11 ], [ 320, 3 ], [ 331, 20 ] ], "source_columns_b": null, "target_columns": [ [ 357, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 37, 4 ] ], "id": "L12.mlp.11", "node": "l:390", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 598, 3 ] ], "source_columns_b": null, "target_columns": [ [ 358, 2 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [], "id": "L12.mlp.12", "node": "l:412", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 360, 2 ] ], "type": "compute_literal_value" }, { "hidden_slots": [ [ 41, 68 ] ], "id": "L12.mlp.13", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 308, 11 ], [ 320, 3 ], [ 331, 20 ] ], "source_columns_b": null, "target_columns": [ [ 308, 11 ], [ 320, 3 ], [ 331, 20 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 109, 68 ] ], "id": "L12.mlp.14", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 385, 34 ] ], "source_columns_b": null, "target_columns": [ [ 385, 34 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 177, 68 ] ], "id": "L12.mlp.15", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 487, 34 ] ], "source_columns_b": null, "target_columns": [ [ 487, 34 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 245, 68 ] ], "id": "L12.mlp.16", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 555, 34 ] ], "source_columns_b": null, "target_columns": [ [ 555, 34 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 313, 68 ] ], "id": "L12.mlp.17", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 57, 2 ], [ 273, 12 ], [ 288, 20 ] ], "source_columns_b": null, "target_columns": [ [ 57, 2 ], [ 273, 12 ], [ 288, 20 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 381, 6 ] ], "id": "L12.mlp.18", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 592, 3 ] ], "source_columns_b": null, "target_columns": [ [ 592, 3 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 387, 6 ] ], "id": "L12.mlp.19", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 595, 3 ] ], "source_columns_b": null, "target_columns": [ [ 595, 3 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 393, 2 ] ], "id": "L12.mlp.20", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 589, 1 ] ], "source_columns_b": null, "target_columns": [ [ 589, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 395, 2 ] ], "id": "L12.mlp.21", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 590, 1 ] ], "source_columns_b": null, "target_columns": [ [ 590, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 397, 2 ] ], "id": "L12.mlp.22", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 591, 1 ] ], "source_columns_b": null, "target_columns": [ [ 591, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:335", "l:308", "l:305", "l:412", "l:390", "l:411", "l:389", "l:346", "l:349", "l:350", "l:353", "l:354", "l:357", "l:345", "l:211", "l:230", "l:210", "l:239", "l:249", "l:263", "l:180", "l:80" ] }, { "active_attention_heads": 2, "active_mlp_neurons": 1888, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L13.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": "l:344", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 209, 1 ], [ 211, 2 ], [ 222, 1 ], [ 256, 1 ], [ 263, 1 ] ], "source_columns_b": null, "target_columns": [ [ 58, 1 ] ], "type": "compute_linear" }, { "heads": [ 1, 1 ], "id": "L13.attn.1", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 211, 1 ], [ 222, 1 ], [ 263, 1 ] ], "type": "cancel" } ], "emitted_attention_heads": 2, "index": 13, "mlp_operations": [ { "hidden_slots": [ [ 1, 400 ] ], "id": "L13.mlp.0", "node": "l:209", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 120, 1 ], [ 355, 1 ] ], "source_columns_b": null, "target_columns": [ [ 57, 1 ], [ 211, 1 ], [ 222, 1 ], [ 263, 1 ], [ 273, 12 ], [ 288, 16 ], [ 305, 2 ], [ 308, 11 ], [ 320, 3 ], [ 331, 11 ], [ 343, 2 ], [ 347, 1 ], [ 350, 1 ], [ 362, 27 ], [ 390, 9 ], [ 404, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 401, 402 ] ], "id": "L13.mlp.1", "node": "l:248", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 327, 1 ] ], "source_columns_b": null, "target_columns": [ [ 342, 1 ], [ 399, 5 ], [ 405, 9 ], [ 415, 2 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 803, 42 ] ], "id": "L13.mlp.2", "node": "l:265", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 327, 1 ] ], "source_columns_b": null, "target_columns": [ [ 417, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 845, 402 ] ], "id": "L13.mlp.3", "node": "l:262", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 207, 1 ] ], "source_columns_b": null, "target_columns": [ [ 434, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 1247, 42 ] ], "id": "L13.mlp.4", "node": "l:269", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 207, 1 ] ], "source_columns_b": null, "target_columns": [ [ 451, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 1289, 2 ] ], "id": "L13.mlp.5", "node": "l:343", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 58, 1 ] ], "source_columns_b": null, "target_columns": [ [ 468, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 1291, 9 ] ], "id": "L13.mlp.6", "node": "l:304", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 598, 3 ], [ 264, 2 ], [ 351, 1 ] ], "source_columns_b": null, "target_columns": [ [ 469, 3 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 1300, 146 ] ], "id": "L13.mlp.7", "node": "l:79", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 357, 1 ] ], "source_columns_b": null, "target_columns": [ [ 472, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 1446, 200 ] ], "id": "L13.mlp.8", "node": "l:388", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 42, 4 ], [ 213, 9 ], [ 223, 1 ], [ 236, 3 ], [ 51, 4 ], [ 67, 1 ], [ 258, 5 ], [ 266, 7 ], [ 358, 2 ] ], "source_columns_b": null, "target_columns": [ [ 345, 1 ], [ 490, 16 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 1646, 200 ] ], "id": "L13.mlp.9", "node": "l:410", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 18, 3 ], [ 55, 2 ], [ 59, 7 ], [ 68, 1 ], [ 106, 1 ], [ 326, 1 ], [ 328, 2 ], [ 21, 5 ], [ 66, 1 ], [ 102, 4 ], [ 107, 4 ], [ 137, 2 ], [ 173, 1 ], [ 360, 2 ] ], "source_columns_b": null, "target_columns": [ [ 346, 1 ], [ 506, 5 ], [ 512, 11 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 1846, 4 ] ], "id": "L13.mlp.10", "node": "l:334", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 352, 3 ] ], "source_columns_b": null, "target_columns": [ [ 523, 2 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [], "id": "L13.mlp.11", "node": "l:302", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 525, 17 ] ], "type": "compute_literal_value" }, { "hidden_slots": [ [ 1850, 6 ] ], "id": "L13.mlp.12", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 598, 3 ] ], "source_columns_b": null, "target_columns": [ [ 598, 3 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1856, 6 ] ], "id": "L13.mlp.13", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 352, 3 ] ], "source_columns_b": null, "target_columns": [ [ 352, 3 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1862, 6 ] ], "id": "L13.mlp.14", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 264, 2 ], [ 351, 1 ] ], "source_columns_b": null, "target_columns": [ [ 264, 2 ], [ 351, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1868, 4 ] ], "id": "L13.mlp.15", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 358, 2 ] ], "source_columns_b": null, "target_columns": [ [ 358, 2 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1872, 2 ] ], "id": "L13.mlp.16", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 209, 1 ] ], "source_columns_b": null, "target_columns": [ [ 209, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1874, 2 ] ], "id": "L13.mlp.17", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 212, 1 ] ], "source_columns_b": null, "target_columns": [ [ 212, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1876, 2 ] ], "id": "L13.mlp.18", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 256, 1 ] ], "source_columns_b": null, "target_columns": [ [ 256, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1878, 2 ] ], "id": "L13.mlp.19", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 120, 1 ] ], "source_columns_b": null, "target_columns": [ [ 120, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1880, 2 ] ], "id": "L13.mlp.20", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 355, 1 ] ], "source_columns_b": null, "target_columns": [ [ 355, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1882, 2 ] ], "id": "L13.mlp.21", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 327, 1 ] ], "source_columns_b": null, "target_columns": [ [ 327, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1884, 2 ] ], "id": "L13.mlp.22", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 207, 1 ] ], "source_columns_b": null, "target_columns": [ [ 207, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1886, 2 ] ], "id": "L13.mlp.23", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 357, 1 ] ], "source_columns_b": null, "target_columns": [ [ 357, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:302", "l:304", "l:334", "l:410", "l:388", "l:333", "l:344", "l:343", "l:342", "l:372", "l:366", "l:359", "l:374", "l:368", "l:209", "l:261", "l:387", "l:409", "l:248", "l:265", "l:262", "l:269", "l:79" ] }, { "active_attention_heads": 2, "active_mlp_neurons": 2254, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L14.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": "l:303", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 469, 3 ] ], "source_columns_b": null, "target_columns": [ [ 120, 1 ], [ 327, 1 ] ], "type": "compute_linear" }, { "heads": [ 1, 1 ], "id": "L14.attn.1", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 360, 2 ] ], "type": "cancel" } ], "emitted_attention_heads": 2, "index": 14, "mlp_operations": [ { "hidden_slots": [ [ 1, 402 ] ], "id": "L14.mlp.0", "node": "l:238", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 356, 1 ] ], "source_columns_b": null, "target_columns": [ [ 207, 1 ], [ 209, 1 ], [ 212, 1 ], [ 256, 1 ], [ 264, 2 ], [ 351, 5 ], [ 357, 5 ], [ 542, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 403, 42 ] ], "id": "L14.mlp.1", "node": "l:257", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 356, 1 ] ], "source_columns_b": null, "target_columns": [ [ 111, 1 ], [ 543, 16 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 445, 402 ] ], "id": "L14.mlp.2", "node": "l:268", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 208, 1 ] ], "source_columns_b": null, "target_columns": [ [ 112, 3 ], [ 348, 1 ], [ 560, 3 ], [ 565, 4 ], [ 570, 4 ], [ 575, 2 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 847, 100 ] ], "id": "L14.mlp.3", "node": "l:260", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 434, 17 ], [ 417, 17 ] ], "source_columns_b": null, "target_columns": [ [ 115, 1 ], [ 577, 2 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 947, 34 ] ], "id": "L14.mlp.4", "node": "l:371", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 468, 1 ], [ 42, 4 ], [ 213, 9 ], [ 223, 1 ], [ 236, 3 ], [ 51, 4 ], [ 67, 1 ], [ 258, 5 ], [ 266, 7 ] ], "source_columns_b": null, "target_columns": [ [ 116, 4 ], [ 349, 1 ], [ 580, 4 ], [ 585, 3 ], [ 601, 5 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 981, 34 ] ], "id": "L14.mlp.5", "node": "l:365", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 468, 1 ], [ 18, 3 ], [ 55, 2 ], [ 59, 7 ], [ 68, 1 ], [ 106, 1 ], [ 326, 1 ], [ 328, 2 ], [ 21, 5 ], [ 66, 1 ], [ 102, 4 ], [ 107, 4 ], [ 137, 2 ], [ 173, 1 ] ], "source_columns_b": null, "target_columns": [ [ 606, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 1015, 34 ] ], "id": "L14.mlp.6", "node": "l:373", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 468, 1 ], [ 51, 4 ], [ 67, 1 ], [ 258, 5 ], [ 266, 7 ], [ 42, 4 ], [ 213, 9 ], [ 223, 1 ], [ 236, 3 ] ], "source_columns_b": null, "target_columns": [ [ 623, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 1049, 34 ] ], "id": "L14.mlp.7", "node": "l:367", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 468, 1 ], [ 21, 5 ], [ 66, 1 ], [ 102, 4 ], [ 107, 4 ], [ 137, 2 ], [ 173, 1 ], [ 18, 3 ], [ 55, 2 ], [ 59, 7 ], [ 68, 1 ], [ 106, 1 ], [ 326, 1 ], [ 328, 2 ] ], "source_columns_b": null, "target_columns": [ [ 640, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 1083, 34 ] ], "id": "L14.mlp.8", "node": "l:341", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 468, 1 ], [ 174, 16 ], [ 210, 1 ], [ 49, 2 ], [ 239, 8 ], [ 248, 3 ], [ 252, 4 ] ], "source_columns_b": null, "target_columns": [ [ 657, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 1117, 34 ] ], "id": "L14.mlp.9", "node": "l:358", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 468, 1 ], [ 49, 2 ], [ 239, 8 ], [ 248, 3 ], [ 252, 4 ], [ 174, 16 ], [ 210, 1 ] ], "source_columns_b": null, "target_columns": [ [ 674, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 1151, 200 ] ], "id": "L14.mlp.10", "node": "l:332", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 174, 16 ], [ 210, 1 ], [ 49, 2 ], [ 239, 8 ], [ 248, 3 ], [ 252, 4 ], [ 523, 2 ] ], "source_columns_b": null, "target_columns": [ [ 691, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 1351, 200 ] ], "id": "L14.mlp.11", "node": "l:300", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 525, 17 ], [ 525, 17 ], [ 120, 1 ], [ 327, 1 ] ], "source_columns_b": null, "target_columns": [ [ 708, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 1551, 2 ] ], "id": "L14.mlp.12", "node": "l:404", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 468, 1 ] ], "source_columns_b": null, "target_columns": [ [ 725, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 1553, 17 ] ], "id": "L14.mlp.13", "node": "l:408", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 33, 1 ], [ 346, 1 ], [ 506, 5 ], [ 512, 11 ] ], "source_columns_b": null, "target_columns": [ [ 726, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 1570, 200 ] ], "id": "L14.mlp.14", "node": "l:208", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 57, 1 ], [ 211, 1 ], [ 222, 1 ], [ 263, 1 ], [ 273, 12 ], [ 288, 16 ], [ 305, 2 ], [ 308, 11 ], [ 320, 3 ], [ 331, 11 ], [ 343, 2 ], [ 347, 1 ], [ 350, 1 ], [ 362, 27 ], [ 390, 9 ], [ 404, 1 ] ], "source_columns_b": null, "target_columns": [ [ 743, 100 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [], "id": "L14.mlp.15", "node": "l:231", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 843, 17 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L14.mlp.16", "node": "l:417", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 860, 2 ] ], "type": "compute_literal_value" }, { "hidden_slots": [], "id": "L14.mlp.17", "node": "l:422", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 862, 2 ] ], "type": "compute_literal_value" }, { "hidden_slots": [ [ 1770, 200 ] ], "id": "L14.mlp.18", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 57, 1 ], [ 211, 1 ], [ 222, 1 ], [ 263, 1 ], [ 273, 12 ], [ 288, 16 ], [ 305, 2 ], [ 308, 11 ], [ 320, 3 ], [ 331, 11 ], [ 343, 2 ], [ 347, 1 ], [ 350, 1 ], [ 362, 27 ], [ 390, 9 ], [ 404, 1 ] ], "source_columns_b": null, "target_columns": [ [ 57, 1 ], [ 211, 1 ], [ 222, 1 ], [ 263, 1 ], [ 273, 12 ], [ 288, 16 ], [ 305, 2 ], [ 308, 11 ], [ 320, 3 ], [ 331, 11 ], [ 343, 2 ], [ 347, 1 ], [ 350, 1 ], [ 362, 27 ], [ 390, 9 ], [ 404, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1970, 34 ] ], "id": "L14.mlp.19", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 174, 16 ], [ 210, 1 ] ], "source_columns_b": null, "target_columns": [ [ 174, 16 ], [ 210, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 2004, 34 ] ], "id": "L14.mlp.20", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 42, 4 ], [ 213, 9 ], [ 223, 1 ], [ 236, 3 ] ], "source_columns_b": null, "target_columns": [ [ 42, 4 ], [ 213, 9 ], [ 223, 1 ], [ 236, 3 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 2038, 34 ] ], "id": "L14.mlp.21", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 18, 3 ], [ 55, 2 ], [ 59, 7 ], [ 68, 1 ], [ 106, 1 ], [ 326, 1 ], [ 328, 2 ] ], "source_columns_b": null, "target_columns": [ [ 18, 3 ], [ 55, 2 ], [ 59, 7 ], [ 68, 1 ], [ 106, 1 ], [ 326, 1 ], [ 328, 2 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 2072, 34 ] ], "id": "L14.mlp.22", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 49, 2 ], [ 239, 8 ], [ 248, 3 ], [ 252, 4 ] ], "source_columns_b": null, "target_columns": [ [ 49, 2 ], [ 239, 8 ], [ 248, 3 ], [ 252, 4 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 2106, 34 ] ], "id": "L14.mlp.23", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 51, 4 ], [ 67, 1 ], [ 258, 5 ], [ 266, 7 ] ], "source_columns_b": null, "target_columns": [ [ 51, 4 ], [ 67, 1 ], [ 258, 5 ], [ 266, 7 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 2140, 34 ] ], "id": "L14.mlp.24", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 21, 5 ], [ 66, 1 ], [ 102, 4 ], [ 107, 4 ], [ 137, 2 ], [ 173, 1 ] ], "source_columns_b": null, "target_columns": [ [ 21, 5 ], [ 66, 1 ], [ 102, 4 ], [ 107, 4 ], [ 137, 2 ], [ 173, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 2174, 34 ] ], "id": "L14.mlp.25", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 525, 17 ] ], "source_columns_b": null, "target_columns": [ [ 525, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 2208, 34 ] ], "id": "L14.mlp.26", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 346, 1 ], [ 506, 5 ], [ 512, 11 ] ], "source_columns_b": null, "target_columns": [ [ 346, 1 ], [ 506, 5 ], [ 512, 11 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 2242, 6 ] ], "id": "L14.mlp.27", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 469, 3 ] ], "source_columns_b": null, "target_columns": [ [ 469, 3 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 2248, 4 ] ], "id": "L14.mlp.28", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 523, 2 ] ], "source_columns_b": null, "target_columns": [ [ 523, 2 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 2252, 2 ] ], "id": "L14.mlp.29", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 58, 1 ] ], "source_columns_b": null, "target_columns": [ [ 58, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:303", "l:332", "l:301", "l:300", "l:341", "l:371", "l:365", "l:358", "l:373", "l:367", "l:364", "l:370", "l:417", "l:416", "l:231", "l:208", "l:247", "l:260", "l:267", "l:422", "l:299", "l:331", "l:408", "l:404", "l:403", "l:238", "l:257", "l:268" ] }, { "active_attention_heads": 6, "active_mlp_neurons": 1274, "attention_operations": [ { "heads": [ 0, 2 ], "id": "L15.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": "l:245", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 743, 100 ] ], "source_columns_b": null, "target_columns": [ [ 52, 3 ], [ 56, 1 ], [ 58, 6 ], [ 65, 1 ], [ 173, 4 ], [ 179, 1 ], [ 326, 1 ] ], "type": "compute_linear" }, { "heads": [ 2, 2 ], "id": "L15.attn.1", "key_source_columns": [ [ 0, 1 ] ], "node": "l:207", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 743, 100 ] ], "source_columns_b": null, "target_columns": [ [ 177, 2 ], [ 180, 9 ], [ 210, 1 ], [ 214, 1 ], [ 217, 1 ], [ 333, 1 ], [ 392, 1 ], [ 404, 1 ] ], "type": "compute_linear" }, { "heads": [ 4, 2 ], "id": "L15.attn.2", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 356, 1 ], [ 468, 1 ], [ 743, 100 ] ], "type": "cancel" } ], "emitted_attention_heads": 6, "index": 15, "mlp_operations": [ { "hidden_slots": [ [ 1, 100 ] ], "id": "L15.mlp.0", "node": "l:236", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 207, 1 ], [ 209, 1 ], [ 212, 1 ], [ 256, 1 ], [ 264, 2 ], [ 351, 5 ], [ 357, 5 ], [ 542, 1 ], [ 52, 3 ], [ 56, 1 ], [ 58, 6 ], [ 65, 1 ], [ 173, 4 ], [ 179, 1 ], [ 326, 1 ] ], "source_columns_b": null, "target_columns": [ [ 42, 1 ], [ 211, 1 ], [ 213, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 101, 100 ] ], "id": "L15.mlp.1", "node": "l:246", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 342, 1 ], [ 399, 5 ], [ 405, 9 ], [ 415, 2 ], [ 111, 1 ], [ 543, 16 ] ], "source_columns_b": null, "target_columns": [ [ 215, 2 ], [ 334, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 201, 100 ] ], "id": "L15.mlp.2", "node": "l:266", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 112, 3 ], [ 348, 1 ], [ 560, 3 ], [ 565, 4 ], [ 570, 4 ], [ 575, 2 ], [ 451, 17 ] ], "source_columns_b": null, "target_columns": [ [ 43, 1 ], [ 218, 2 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 301, 100 ] ], "id": "L15.mlp.3", "node": "l:205", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 177, 2 ], [ 180, 9 ], [ 210, 1 ], [ 214, 1 ], [ 217, 1 ], [ 333, 1 ], [ 392, 1 ], [ 404, 1 ], [ 843, 17 ] ], "source_columns_b": null, "target_columns": [ [ 44, 1 ], [ 220, 1 ], [ 222, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 401, 100 ] ], "id": "L15.mlp.4", "node": "l:363", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 606, 17 ], [ 640, 17 ] ], "source_columns_b": null, "target_columns": [ [ 45, 1 ], [ 223, 1 ], [ 239, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 501, 100 ] ], "id": "L15.mlp.5", "node": "l:369", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 116, 4 ], [ 349, 1 ], [ 580, 4 ], [ 585, 3 ], [ 601, 5 ], [ 623, 17 ] ], "source_columns_b": null, "target_columns": [ [ 236, 3 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 601, 17 ] ], "id": "L15.mlp.6", "node": "l:298", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 33, 1 ], [ 708, 17 ] ], "source_columns_b": null, "target_columns": [ [ 49, 2 ], [ 240, 7 ], [ 248, 3 ], [ 252, 4 ], [ 335, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 618, 17 ] ], "id": "L15.mlp.7", "node": "l:330", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 33, 1 ], [ 691, 17 ] ], "source_columns_b": null, "target_columns": [ [ 51, 1 ], [ 55, 1 ], [ 67, 1 ], [ 258, 6 ], [ 266, 6 ], [ 336, 1 ], [ 391, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 635, 17 ] ], "id": "L15.mlp.8", "node": "l:386", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 33, 1 ], [ 345, 1 ], [ 490, 16 ] ], "source_columns_b": null, "target_columns": [ [ 57, 1 ], [ 273, 12 ], [ 288, 2 ], [ 337, 1 ], [ 339, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 652, 200 ] ], "id": "L15.mlp.9", "node": "l:415", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 606, 17 ], [ 640, 17 ], [ 860, 2 ] ], "source_columns_b": null, "target_columns": [ [ 290, 14 ], [ 305, 2 ], [ 393, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 852, 200 ] ], "id": "L15.mlp.10", "node": "l:420", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 177, 2 ], [ 180, 9 ], [ 210, 1 ], [ 214, 1 ], [ 217, 1 ], [ 333, 1 ], [ 392, 1 ], [ 404, 1 ], [ 843, 17 ], [ 862, 2 ] ], "source_columns_b": null, "target_columns": [ [ 64, 1 ], [ 308, 11 ], [ 320, 3 ], [ 328, 1 ], [ 338, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 1052, 2 ] ], "id": "L15.mlp.11", "node": "l:402", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 35, 1 ], [ 725, 1 ] ], "source_columns_b": null, "target_columns": [ [ 68, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 1054, 34 ] ], "id": "L15.mlp.12", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 345, 1 ], [ 490, 16 ] ], "source_columns_b": null, "target_columns": [ [ 345, 1 ], [ 490, 16 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1088, 34 ] ], "id": "L15.mlp.13", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 691, 17 ] ], "source_columns_b": null, "target_columns": [ [ 691, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1122, 34 ] ], "id": "L15.mlp.14", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 708, 17 ] ], "source_columns_b": null, "target_columns": [ [ 708, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1156, 34 ] ], "id": "L15.mlp.15", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 606, 17 ] ], "source_columns_b": null, "target_columns": [ [ 606, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1190, 34 ] ], "id": "L15.mlp.16", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 640, 17 ] ], "source_columns_b": null, "target_columns": [ [ 640, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1224, 34 ] ], "id": "L15.mlp.17", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 843, 17 ] ], "source_columns_b": null, "target_columns": [ [ 843, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1258, 4 ] ], "id": "L15.mlp.18", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 120, 1 ], [ 327, 1 ] ], "source_columns_b": null, "target_columns": [ [ 120, 1 ], [ 327, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1262, 4 ] ], "id": "L15.mlp.19", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 860, 2 ] ], "source_columns_b": null, "target_columns": [ [ 860, 2 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1266, 4 ] ], "id": "L15.mlp.20", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 862, 2 ] ], "source_columns_b": null, "target_columns": [ [ 862, 2 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1270, 2 ] ], "id": "L15.mlp.21", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 33, 1 ] ], "source_columns_b": null, "target_columns": [ [ 33, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1272, 2 ] ], "id": "L15.mlp.22", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 725, 1 ] ], "source_columns_b": null, "target_columns": [ [ 725, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:363", "l:369", "l:362", "l:415", "l:207", "l:245", "l:206", "l:205", "l:237", "l:236", "l:246", "l:266", "l:289", "l:235", "l:291", "l:259", "l:421", "l:420", "l:298", "l:330", "l:386", "l:414", "l:419", "l:402" ] }, { "active_attention_heads": 4, "active_mlp_neurons": 792, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L16.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": "l:395", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 45, 1 ], [ 223, 1 ], [ 239, 1 ] ], "source_columns_b": null, "target_columns": [ [ 33, 1 ], [ 327, 1 ] ], "type": "compute_linear" }, { "heads": [ 1, 1 ], "id": "L16.attn.1", "key_source_columns": [ [ 0, 1 ] ], "node": "l:400", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 44, 1 ], [ 220, 1 ], [ 222, 1 ] ], "source_columns_b": null, "target_columns": [ [ 68, 1 ], [ 356, 1 ] ], "type": "compute_linear" }, { "heads": [ 2, 1 ], "id": "L16.attn.2", "key_source_columns": [ [ 0, 1 ] ], "node": "l:401", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 68, 1 ] ], "source_columns_b": null, "target_columns": [ [ 364, 1 ] ], "type": "compute_linear" }, { "heads": [ 3, 1 ], "id": "L16.attn.3", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 68, 1 ] ], "type": "cancel" } ], "emitted_attention_heads": 4, "index": 16, "mlp_operations": [ { "hidden_slots": [ [ 1, 9 ] ], "id": "L16.mlp.0", "node": "l:234", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 42, 1 ], [ 211, 1 ], [ 213, 1 ], [ 215, 2 ], [ 334, 1 ] ], "source_columns_b": null, "target_columns": [ [ 340, 1 ], [ 362, 2 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 10, 9 ] ], "id": "L16.mlp.1", "node": "l:258", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 115, 1 ], [ 577, 2 ], [ 43, 1 ], [ 218, 2 ] ], "source_columns_b": null, "target_columns": [ [ 365, 3 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 19, 9 ] ], "id": "L16.mlp.2", "node": "l:288", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 44, 1 ], [ 220, 1 ], [ 222, 1 ], [ 42, 1 ], [ 211, 1 ], [ 213, 1 ] ], "source_columns_b": null, "target_columns": [ [ 368, 3 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 28, 9 ] ], "id": "L16.mlp.3", "node": "l:290", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 215, 2 ], [ 334, 1 ], [ 115, 1 ], [ 577, 2 ] ], "source_columns_b": null, "target_columns": [ [ 371, 3 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 37, 9 ] ], "id": "L16.mlp.4", "node": "l:361", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 45, 1 ], [ 223, 1 ], [ 239, 1 ], [ 236, 3 ] ], "source_columns_b": null, "target_columns": [ [ 374, 3 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 46, 42 ] ], "id": "L16.mlp.5", "node": "l:179", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 208, 1 ] ], "source_columns_b": null, "target_columns": [ [ 341, 1 ], [ 343, 1 ], [ 377, 12 ], [ 390, 1 ], [ 394, 1 ], [ 398, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 88, 200 ] ], "id": "L16.mlp.6", "node": "l:393", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 116, 4 ], [ 349, 1 ], [ 580, 4 ], [ 585, 3 ], [ 601, 5 ], [ 623, 17 ], [ 33, 1 ], [ 327, 1 ] ], "source_columns_b": null, "target_columns": [ [ 18, 2 ], [ 329, 1 ], [ 344, 2 ], [ 468, 4 ], [ 490, 8 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 288, 200 ] ], "id": "L16.mlp.7", "node": "l:398", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 207, 1 ], [ 209, 1 ], [ 212, 1 ], [ 256, 1 ], [ 264, 2 ], [ 351, 5 ], [ 357, 5 ], [ 542, 1 ], [ 52, 3 ], [ 56, 1 ], [ 58, 6 ], [ 65, 1 ], [ 173, 4 ], [ 179, 1 ], [ 326, 1 ], [ 68, 1 ], [ 356, 1 ] ], "source_columns_b": null, "target_columns": [ [ 20, 1 ], [ 102, 1 ], [ 346, 1 ], [ 498, 4 ], [ 505, 6 ], [ 512, 3 ], [ 519, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 488, 17 ] ], "id": "L16.mlp.8", "node": "l:413", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 35, 1 ], [ 290, 14 ], [ 305, 2 ], [ 393, 1 ] ], "source_columns_b": null, "target_columns": [ [ 21, 1 ], [ 103, 3 ], [ 515, 4 ], [ 520, 2 ], [ 525, 7 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 505, 17 ] ], "id": "L16.mlp.9", "node": "l:418", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 40, 1 ], [ 64, 1 ], [ 308, 11 ], [ 320, 3 ], [ 328, 1 ], [ 338, 1 ] ], "source_columns_b": null, "target_columns": [ [ 22, 2 ], [ 106, 5 ], [ 120, 1 ], [ 347, 1 ], [ 395, 1 ], [ 532, 2 ], [ 535, 3 ], [ 540, 2 ] ], "type": "compute_ffn" }, { "hidden_slots": [], "id": "L16.mlp.10", "node": "l:401", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 364, 1 ] ], "type": "compute_bias" }, { "hidden_slots": [ [ 522, 34 ] ], "id": "L16.mlp.11", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 116, 4 ], [ 349, 1 ], [ 580, 4 ], [ 585, 3 ], [ 601, 5 ] ], "source_columns_b": null, "target_columns": [ [ 116, 4 ], [ 349, 1 ], [ 580, 4 ], [ 585, 3 ], [ 601, 5 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 556, 34 ] ], "id": "L16.mlp.12", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 623, 17 ] ], "source_columns_b": null, "target_columns": [ [ 623, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 590, 34 ] ], "id": "L16.mlp.13", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 290, 14 ], [ 305, 2 ], [ 393, 1 ] ], "source_columns_b": null, "target_columns": [ [ 290, 14 ], [ 305, 2 ], [ 393, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 624, 34 ] ], "id": "L16.mlp.14", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 177, 2 ], [ 180, 9 ], [ 210, 1 ], [ 214, 1 ], [ 217, 1 ], [ 333, 1 ], [ 392, 1 ], [ 404, 1 ] ], "source_columns_b": null, "target_columns": [ [ 177, 2 ], [ 180, 9 ], [ 210, 1 ], [ 214, 1 ], [ 217, 1 ], [ 333, 1 ], [ 392, 1 ], [ 404, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 658, 34 ] ], "id": "L16.mlp.15", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 52, 3 ], [ 56, 1 ], [ 58, 6 ], [ 65, 1 ], [ 173, 4 ], [ 179, 1 ], [ 326, 1 ] ], "source_columns_b": null, "target_columns": [ [ 52, 3 ], [ 56, 1 ], [ 58, 6 ], [ 65, 1 ], [ 173, 4 ], [ 179, 1 ], [ 326, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 692, 34 ] ], "id": "L16.mlp.16", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 64, 1 ], [ 308, 11 ], [ 320, 3 ], [ 328, 1 ], [ 338, 1 ] ], "source_columns_b": null, "target_columns": [ [ 64, 1 ], [ 308, 11 ], [ 320, 3 ], [ 328, 1 ], [ 338, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 726, 34 ] ], "id": "L16.mlp.17", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 207, 1 ], [ 209, 1 ], [ 212, 1 ], [ 256, 1 ], [ 264, 2 ], [ 351, 5 ], [ 357, 5 ], [ 542, 1 ] ], "source_columns_b": null, "target_columns": [ [ 207, 1 ], [ 209, 1 ], [ 212, 1 ], [ 256, 1 ], [ 264, 2 ], [ 351, 5 ], [ 357, 5 ], [ 542, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 760, 6 ] ], "id": "L16.mlp.18", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 45, 1 ], [ 223, 1 ], [ 239, 1 ] ], "source_columns_b": null, "target_columns": [ [ 45, 1 ], [ 223, 1 ], [ 239, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 766, 6 ] ], "id": "L16.mlp.19", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 236, 3 ] ], "source_columns_b": null, "target_columns": [ [ 236, 3 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 772, 6 ] ], "id": "L16.mlp.20", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 42, 1 ], [ 211, 1 ], [ 213, 1 ] ], "source_columns_b": null, "target_columns": [ [ 42, 1 ], [ 211, 1 ], [ 213, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 778, 6 ] ], "id": "L16.mlp.21", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 215, 2 ], [ 334, 1 ] ], "source_columns_b": null, "target_columns": [ [ 215, 2 ], [ 334, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 784, 6 ] ], "id": "L16.mlp.22", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 115, 1 ], [ 577, 2 ] ], "source_columns_b": null, "target_columns": [ [ 115, 1 ], [ 577, 2 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 790, 2 ] ], "id": "L16.mlp.23", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 208, 1 ] ], "source_columns_b": null, "target_columns": [ [ 208, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:361", "l:395", "l:394", "l:393", "l:288", "l:234", "l:290", "l:258", "l:322", "l:287", "l:233", "l:400", "l:399", "l:398", "l:392", "l:397", "l:413", "l:418", "l:407", "l:401", "l:179" ] }, { "active_attention_heads": 4, "active_mlp_neurons": 696, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L17.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": "l:360", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 374, 3 ] ], "source_columns_b": null, "target_columns": [ [ 65, 1 ], [ 326, 1 ] ], "type": "compute_linear" }, { "heads": [ 1, 1 ], "id": "L17.attn.1", "key_source_columns": [ [ 0, 1 ] ], "node": "l:379", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 368, 3 ] ], "source_columns_b": null, "target_columns": [ [ 43, 1 ], [ 60, 1 ] ], "type": "compute_linear" }, { "heads": [ 2, 1 ], "id": "L17.attn.2", "key_source_columns": [ [ 0, 1 ] ], "node": "l:406", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 726, 17 ], [ 21, 1 ], [ 103, 3 ], [ 515, 4 ], [ 520, 2 ], [ 525, 7 ], [ 22, 2 ], [ 106, 5 ], [ 120, 1 ], [ 347, 1 ], [ 395, 1 ], [ 532, 2 ], [ 535, 3 ], [ 540, 2 ] ], "source_columns_b": null, "target_columns": [ [ 33, 1 ], [ 173, 15 ], [ 392, 1 ] ], "type": "compute_linear" }, { "heads": [ 3, 1 ], "id": "L17.attn.3", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 21, 3 ], [ 33, 1 ], [ 43, 1 ], [ 68, 1 ], [ 103, 8 ], [ 120, 1 ], [ 218, 2 ], [ 327, 1 ], [ 347, 1 ], [ 356, 1 ], [ 395, 1 ], [ 515, 4 ], [ 520, 2 ], [ 525, 9 ], [ 535, 3 ], [ 540, 2 ], [ 726, 17 ] ], "type": "cancel" } ], "emitted_attention_heads": 4, "index": 17, "mlp_operations": [ { "hidden_slots": [ [ 1, 9 ] ], "id": "L17.mlp.0", "node": "l:232", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 340, 1 ], [ 362, 2 ], [ 365, 3 ] ], "source_columns_b": null, "target_columns": [ [ 188, 1 ], [ 327, 1 ], [ 333, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 10, 9 ] ], "id": "L17.mlp.1", "node": "l:321", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 44, 1 ], [ 220, 1 ], [ 222, 1 ], [ 340, 1 ], [ 362, 2 ] ], "source_columns_b": null, "target_columns": [ [ 208, 1 ], [ 334, 1 ], [ 404, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 19, 9 ] ], "id": "L17.mlp.2", "node": "l:286", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 368, 6 ] ], "source_columns_b": null, "target_columns": [ [ 207, 1 ], [ 209, 1 ], [ 211, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 28, 200 ] ], "id": "L17.mlp.3", "node": "l:339", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 657, 34 ], [ 65, 1 ], [ 326, 1 ] ], "source_columns_b": null, "target_columns": [ [ 42, 1 ], [ 45, 1 ], [ 210, 1 ], [ 212, 8 ], [ 223, 1 ], [ 236, 4 ], [ 256, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 228, 200 ] ], "id": "L17.mlp.4", "node": "l:377", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 342, 1 ], [ 399, 5 ], [ 405, 9 ], [ 415, 2 ], [ 111, 1 ], [ 543, 16 ], [ 43, 1 ], [ 60, 1 ] ], "source_columns_b": null, "target_columns": [ [ 52, 3 ], [ 56, 1 ], [ 58, 2 ], [ 264, 2 ], [ 290, 9 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 428, 17 ] ], "id": "L17.mlp.5", "node": "l:391", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 35, 1 ], [ 18, 2 ], [ 329, 1 ], [ 344, 2 ], [ 468, 4 ], [ 490, 8 ] ], "source_columns_b": null, "target_columns": [ [ 61, 2 ], [ 299, 5 ], [ 305, 2 ], [ 308, 4 ], [ 313, 1 ], [ 315, 1 ], [ 338, 1 ], [ 393, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 445, 17 ] ], "id": "L17.mlp.6", "node": "l:396", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 40, 1 ], [ 20, 1 ], [ 102, 1 ], [ 346, 1 ], [ 498, 4 ], [ 505, 6 ], [ 512, 3 ], [ 519, 1 ] ], "source_columns_b": null, "target_columns": [ [ 63, 2 ], [ 68, 1 ], [ 316, 1 ], [ 318, 1 ], [ 320, 3 ], [ 328, 1 ], [ 351, 1 ], [ 354, 1 ], [ 356, 6 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 462, 34 ] ], "id": "L17.mlp.7", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 657, 17 ] ], "source_columns_b": null, "target_columns": [ [ 657, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 496, 34 ] ], "id": "L17.mlp.8", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 674, 17 ] ], "source_columns_b": null, "target_columns": [ [ 674, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 530, 34 ] ], "id": "L17.mlp.9", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 18, 2 ], [ 329, 1 ], [ 344, 2 ], [ 468, 4 ], [ 490, 8 ] ], "source_columns_b": null, "target_columns": [ [ 18, 2 ], [ 329, 1 ], [ 344, 2 ], [ 468, 4 ], [ 490, 8 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 564, 34 ] ], "id": "L17.mlp.10", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 20, 1 ], [ 102, 1 ], [ 346, 1 ], [ 498, 4 ], [ 505, 6 ], [ 512, 3 ], [ 519, 1 ] ], "source_columns_b": null, "target_columns": [ [ 20, 1 ], [ 102, 1 ], [ 346, 1 ], [ 498, 4 ], [ 505, 6 ], [ 512, 3 ], [ 519, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 598, 34 ] ], "id": "L17.mlp.11", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 111, 1 ], [ 543, 16 ] ], "source_columns_b": null, "target_columns": [ [ 111, 1 ], [ 543, 16 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 632, 34 ] ], "id": "L17.mlp.12", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 342, 1 ], [ 399, 5 ], [ 405, 9 ], [ 415, 2 ] ], "source_columns_b": null, "target_columns": [ [ 342, 1 ], [ 399, 5 ], [ 405, 9 ], [ 415, 2 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 666, 6 ] ], "id": "L17.mlp.13", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 374, 3 ] ], "source_columns_b": null, "target_columns": [ [ 374, 3 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 672, 6 ] ], "id": "L17.mlp.14", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 368, 3 ] ], "source_columns_b": null, "target_columns": [ [ 368, 3 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 678, 6 ] ], "id": "L17.mlp.15", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 340, 1 ], [ 362, 2 ] ], "source_columns_b": null, "target_columns": [ [ 340, 1 ], [ 362, 2 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 684, 6 ] ], "id": "L17.mlp.16", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 371, 3 ] ], "source_columns_b": null, "target_columns": [ [ 371, 3 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 690, 6 ] ], "id": "L17.mlp.17", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 365, 3 ] ], "source_columns_b": null, "target_columns": [ [ 365, 3 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:360", "l:340", "l:339", "l:321", "l:286", "l:232", "l:204", "l:379", "l:378", "l:377", "l:338", "l:376", "l:391", "l:396", "l:385", "l:406", "l:502" ] }, { "active_attention_heads": 4, "active_mlp_neurons": 708, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L18.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": "l:320", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 208, 1 ], [ 334, 1 ], [ 404, 1 ] ], "source_columns_b": null, "target_columns": [ [ 65, 1 ], [ 326, 1 ] ], "type": "compute_linear" }, { "heads": [ 1, 1 ], "id": "L18.attn.1", "key_source_columns": [ [ 0, 1 ] ], "node": "l:285", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 207, 1 ], [ 209, 1 ], [ 211, 1 ] ], "source_columns_b": null, "target_columns": [ [ 43, 1 ], [ 273, 1 ] ], "type": "compute_linear" }, { "heads": [ 2, 1 ], "id": "L18.attn.2", "key_source_columns": [ [ 0, 1 ] ], "node": "l:384", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 57, 1 ], [ 273, 12 ], [ 288, 2 ], [ 337, 1 ], [ 339, 1 ], [ 61, 2 ], [ 299, 5 ], [ 305, 2 ], [ 308, 4 ], [ 313, 1 ], [ 315, 1 ], [ 338, 1 ], [ 393, 1 ], [ 63, 2 ], [ 68, 1 ], [ 316, 1 ], [ 318, 1 ], [ 320, 3 ], [ 328, 1 ], [ 351, 1 ], [ 354, 1 ], [ 356, 6 ] ], "source_columns_b": null, "target_columns": [ [ 57, 1 ], [ 274, 11 ], [ 288, 2 ], [ 302, 1 ], [ 337, 1 ], [ 339, 1 ] ], "type": "compute_linear" }, { "heads": [ 3, 1 ], "id": "L18.attn.3", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 43, 1 ], [ 57, 1 ], [ 60, 3 ], [ 65, 1 ], [ 273, 12 ], [ 288, 2 ], [ 299, 5 ], [ 305, 2 ], [ 308, 4 ], [ 313, 1 ], [ 315, 1 ], [ 326, 1 ], [ 337, 3 ], [ 393, 1 ] ], "type": "cancel" } ], "emitted_attention_heads": 4, "index": 18, "mlp_operations": [ { "hidden_slots": [ [ 1, 9 ] ], "id": "L18.mlp.0", "node": "l:203", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 44, 1 ], [ 220, 1 ], [ 222, 1 ], [ 188, 1 ], [ 327, 1 ], [ 333, 1 ] ], "source_columns_b": null, "target_columns": [ [ 299, 3 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 10, 200 ] ], "id": "L18.mlp.1", "node": "l:318", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 434, 17 ], [ 417, 17 ], [ 65, 1 ], [ 326, 1 ] ], "source_columns_b": null, "target_columns": [ [ 60, 3 ], [ 303, 1 ], [ 305, 2 ], [ 308, 4 ], [ 313, 1 ], [ 315, 1 ], [ 338, 1 ], [ 340, 1 ], [ 362, 2 ], [ 393, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 210, 200 ] ], "id": "L18.mlp.2", "node": "l:283", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 112, 3 ], [ 348, 1 ], [ 560, 3 ], [ 565, 4 ], [ 570, 4 ], [ 575, 2 ], [ 451, 17 ], [ 43, 1 ], [ 273, 1 ] ], "source_columns_b": null, "target_columns": [ [ 365, 12 ], [ 399, 5 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 410, 17 ] ], "id": "L18.mlp.3", "node": "l:337", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 35, 1 ], [ 42, 1 ], [ 45, 1 ], [ 210, 1 ], [ 212, 8 ], [ 223, 1 ], [ 236, 4 ], [ 256, 1 ] ], "source_columns_b": null, "target_columns": [ [ 18, 1 ], [ 329, 1 ], [ 342, 1 ], [ 344, 1 ], [ 405, 9 ], [ 415, 2 ], [ 468, 2 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 427, 17 ] ], "id": "L18.mlp.4", "node": "l:375", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 40, 1 ], [ 52, 3 ], [ 56, 1 ], [ 58, 2 ], [ 264, 2 ], [ 290, 9 ] ], "source_columns_b": null, "target_columns": [ [ 19, 1 ], [ 345, 1 ], [ 470, 2 ], [ 490, 12 ], [ 510, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 444, 34 ] ], "id": "L18.mlp.5", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 42, 1 ], [ 45, 1 ], [ 210, 1 ], [ 212, 8 ], [ 223, 1 ], [ 236, 4 ], [ 256, 1 ] ], "source_columns_b": null, "target_columns": [ [ 42, 1 ], [ 45, 1 ], [ 210, 1 ], [ 212, 8 ], [ 223, 1 ], [ 236, 4 ], [ 256, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 478, 34 ] ], "id": "L18.mlp.6", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 52, 3 ], [ 56, 1 ], [ 58, 2 ], [ 264, 2 ], [ 290, 9 ] ], "source_columns_b": null, "target_columns": [ [ 52, 3 ], [ 56, 1 ], [ 58, 2 ], [ 264, 2 ], [ 290, 9 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 512, 34 ] ], "id": "L18.mlp.7", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 63, 2 ], [ 68, 1 ], [ 316, 1 ], [ 318, 1 ], [ 320, 3 ], [ 328, 1 ], [ 351, 1 ], [ 354, 1 ], [ 356, 6 ] ], "source_columns_b": null, "target_columns": [ [ 63, 2 ], [ 68, 1 ], [ 316, 1 ], [ 318, 1 ], [ 320, 3 ], [ 328, 1 ], [ 351, 1 ], [ 354, 1 ], [ 356, 6 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 546, 34 ] ], "id": "L18.mlp.8", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 417, 17 ] ], "source_columns_b": null, "target_columns": [ [ 417, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 580, 34 ] ], "id": "L18.mlp.9", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 434, 17 ] ], "source_columns_b": null, "target_columns": [ [ 434, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 614, 34 ] ], "id": "L18.mlp.10", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 451, 17 ] ], "source_columns_b": null, "target_columns": [ [ 451, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 648, 34 ] ], "id": "L18.mlp.11", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 112, 3 ], [ 348, 1 ], [ 560, 3 ], [ 565, 4 ], [ 570, 4 ], [ 575, 2 ] ], "source_columns_b": null, "target_columns": [ [ 112, 3 ], [ 348, 1 ], [ 560, 3 ], [ 565, 4 ], [ 570, 4 ], [ 575, 2 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 682, 6 ] ], "id": "L18.mlp.12", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 44, 1 ], [ 220, 1 ], [ 222, 1 ] ], "source_columns_b": null, "target_columns": [ [ 44, 1 ], [ 220, 1 ], [ 222, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 688, 6 ] ], "id": "L18.mlp.13", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 208, 1 ], [ 334, 1 ], [ 404, 1 ] ], "source_columns_b": null, "target_columns": [ [ 208, 1 ], [ 334, 1 ], [ 404, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 694, 6 ] ], "id": "L18.mlp.14", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 207, 1 ], [ 209, 1 ], [ 211, 1 ] ], "source_columns_b": null, "target_columns": [ [ 207, 1 ], [ 209, 1 ], [ 211, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 700, 6 ] ], "id": "L18.mlp.15", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 188, 1 ], [ 327, 1 ], [ 333, 1 ] ], "source_columns_b": null, "target_columns": [ [ 188, 1 ], [ 327, 1 ], [ 333, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 706, 2 ] ], "id": "L18.mlp.16", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 35, 1 ] ], "source_columns_b": null, "target_columns": [ [ 35, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:203", "l:320", "l:285", "l:319", "l:318", "l:284", "l:283", "l:282", "l:317", "l:337", "l:375", "l:329", "l:384", "l:492" ] }, { "active_attention_heads": 3, "active_mlp_neurons": 446, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L19.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": "l:202", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 299, 3 ] ], "source_columns_b": null, "target_columns": [ [ 65, 1 ], [ 326, 1 ] ], "type": "compute_linear" }, { "heads": [ 1, 1 ], "id": "L19.attn.1", "key_source_columns": [ [ 0, 1 ] ], "node": "l:328", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 51, 1 ], [ 55, 1 ], [ 67, 1 ], [ 258, 6 ], [ 266, 6 ], [ 336, 1 ], [ 391, 1 ], [ 18, 1 ], [ 329, 1 ], [ 342, 1 ], [ 344, 1 ], [ 405, 9 ], [ 415, 2 ], [ 468, 2 ], [ 19, 1 ], [ 345, 1 ], [ 470, 2 ], [ 490, 12 ], [ 510, 1 ] ], "source_columns_b": null, "target_columns": [ [ 35, 1 ], [ 42, 2 ], [ 188, 1 ], [ 207, 9 ], [ 327, 1 ], [ 333, 2 ], [ 404, 1 ] ], "type": "compute_linear" }, { "heads": [ 2, 1 ], "id": "L19.attn.2", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 43, 1 ], [ 51, 1 ], [ 55, 1 ], [ 65, 1 ], [ 67, 1 ], [ 258, 6 ], [ 266, 6 ], [ 273, 1 ], [ 326, 1 ], [ 336, 1 ], [ 391, 1 ] ], "type": "cancel" } ], "emitted_attention_heads": 3, "index": 19, "mlp_operations": [ { "hidden_slots": [ [ 1, 200 ] ], "id": "L19.mlp.0", "node": "l:77", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 472, 17 ], [ 341, 1 ], [ 343, 1 ], [ 377, 12 ], [ 390, 1 ], [ 394, 1 ], [ 398, 1 ], [ 65, 1 ], [ 326, 1 ] ], "source_columns_b": null, "target_columns": [ [ 44, 2 ], [ 51, 1 ], [ 216, 5 ], [ 222, 2 ], [ 236, 4 ], [ 256, 1 ], [ 262, 2 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 201, 17 ] ], "id": "L19.mlp.1", "node": "l:281", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 40, 1 ], [ 365, 12 ], [ 399, 5 ] ], "source_columns_b": null, "target_columns": [ [ 52, 4 ], [ 67, 1 ], [ 258, 4 ], [ 264, 3 ], [ 268, 3 ], [ 273, 1 ], [ 391, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 218, 17 ] ], "id": "L19.mlp.2", "node": "l:316", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 40, 1 ], [ 60, 3 ], [ 303, 1 ], [ 305, 2 ], [ 308, 4 ], [ 313, 1 ], [ 315, 1 ], [ 338, 1 ], [ 340, 1 ], [ 362, 2 ], [ 393, 1 ] ], "source_columns_b": null, "target_columns": [ [ 56, 1 ], [ 58, 2 ], [ 63, 1 ], [ 290, 9 ], [ 316, 1 ], [ 318, 1 ], [ 320, 1 ], [ 328, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 235, 1 ] ], "id": "L19.mlp.3", "node": "l:327", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 35, 1 ], [ 42, 2 ], [ 188, 1 ], [ 207, 9 ], [ 327, 1 ], [ 333, 2 ], [ 404, 1 ] ], "source_columns_b": null, "target_columns": [ [ 321, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 236, 34 ] ], "id": "L19.mlp.4", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 60, 3 ], [ 303, 1 ], [ 305, 2 ], [ 308, 4 ], [ 313, 1 ], [ 315, 1 ], [ 338, 1 ], [ 340, 1 ], [ 362, 2 ], [ 393, 1 ] ], "source_columns_b": null, "target_columns": [ [ 60, 3 ], [ 303, 1 ], [ 305, 2 ], [ 308, 4 ], [ 313, 1 ], [ 315, 1 ], [ 338, 1 ], [ 340, 1 ], [ 362, 2 ], [ 393, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 270, 34 ] ], "id": "L19.mlp.5", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 365, 12 ], [ 399, 5 ] ], "source_columns_b": null, "target_columns": [ [ 365, 12 ], [ 399, 5 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 304, 34 ] ], "id": "L19.mlp.6", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 18, 1 ], [ 329, 1 ], [ 342, 1 ], [ 344, 1 ], [ 405, 9 ], [ 415, 2 ], [ 468, 2 ] ], "source_columns_b": null, "target_columns": [ [ 18, 1 ], [ 329, 1 ], [ 342, 1 ], [ 344, 1 ], [ 405, 9 ], [ 415, 2 ], [ 468, 2 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 338, 34 ] ], "id": "L19.mlp.7", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 19, 1 ], [ 345, 1 ], [ 470, 2 ], [ 490, 12 ], [ 510, 1 ] ], "source_columns_b": null, "target_columns": [ [ 19, 1 ], [ 345, 1 ], [ 470, 2 ], [ 490, 12 ], [ 510, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 372, 34 ] ], "id": "L19.mlp.8", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 341, 1 ], [ 343, 1 ], [ 377, 12 ], [ 390, 1 ], [ 394, 1 ], [ 398, 1 ] ], "source_columns_b": null, "target_columns": [ [ 341, 1 ], [ 343, 1 ], [ 377, 12 ], [ 390, 1 ], [ 394, 1 ], [ 398, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 406, 34 ] ], "id": "L19.mlp.9", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 472, 17 ] ], "source_columns_b": null, "target_columns": [ [ 472, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 440, 6 ] ], "id": "L19.mlp.10", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 299, 3 ] ], "source_columns_b": null, "target_columns": [ [ 299, 3 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:202", "l:78", "l:77", "l:70", "l:281", "l:276", "l:316", "l:297", "l:328", "l:327", "l:482" ] }, { "active_attention_heads": 4, "active_mlp_neurons": 128, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L20.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": "l:275", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 156, 17 ], [ 29, 4 ], [ 34, 1 ], [ 36, 4 ], [ 190, 8 ], [ 52, 4 ], [ 67, 1 ], [ 258, 4 ], [ 264, 3 ], [ 268, 3 ], [ 273, 1 ], [ 391, 1 ] ], "source_columns_b": null, "target_columns": [ [ 18, 1 ], [ 29, 4 ], [ 34, 1 ], [ 65, 1 ], [ 156, 8 ], [ 326, 1 ], [ 329, 1 ] ], "type": "compute_linear" }, { "heads": [ 1, 1 ], "id": "L20.attn.1", "key_source_columns": [ [ 0, 1 ] ], "node": "l:296", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 49, 2 ], [ 240, 7 ], [ 248, 3 ], [ 252, 4 ], [ 335, 1 ], [ 46, 1 ], [ 198, 9 ], [ 224, 7 ], [ 56, 1 ], [ 58, 2 ], [ 63, 1 ], [ 290, 9 ], [ 316, 1 ], [ 318, 1 ], [ 320, 1 ], [ 328, 1 ] ], "source_columns_b": null, "target_columns": [ [ 36, 3 ], [ 164, 9 ], [ 190, 4 ], [ 197, 1 ] ], "type": "compute_linear" }, { "heads": [ 2, 2 ], "id": "L20.attn.2", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 29, 4 ], [ 34, 1 ], [ 36, 4 ], [ 49, 2 ], [ 56, 1 ], [ 58, 2 ], [ 63, 1 ], [ 65, 1 ], [ 156, 17 ], [ 190, 8 ], [ 240, 7 ], [ 248, 3 ], [ 252, 4 ], [ 290, 9 ], [ 316, 1 ], [ 318, 1 ], [ 320, 1 ], [ 326, 1 ], [ 328, 1 ], [ 335, 1 ] ], "type": "cancel" } ], "emitted_attention_heads": 4, "index": 20, "mlp_operations": [ { "hidden_slots": [ [ 1, 17 ] ], "id": "L20.mlp.0", "node": "l:69", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 40, 1 ], [ 44, 2 ], [ 51, 1 ], [ 216, 5 ], [ 222, 2 ], [ 236, 4 ], [ 256, 1 ], [ 262, 2 ] ], "source_columns_b": null, "target_columns": [ [ 39, 1 ], [ 49, 1 ], [ 194, 3 ], [ 240, 7 ], [ 248, 3 ], [ 252, 1 ], [ 335, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 18, 1 ] ], "id": "L20.mlp.1", "node": "l:274", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 18, 1 ], [ 29, 4 ], [ 34, 1 ], [ 65, 1 ], [ 156, 8 ], [ 326, 1 ], [ 329, 1 ] ], "source_columns_b": null, "target_columns": [ [ 50, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 19, 1 ] ], "id": "L20.mlp.2", "node": "l:295", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 36, 3 ], [ 164, 9 ], [ 190, 4 ], [ 197, 1 ] ], "source_columns_b": null, "target_columns": [ [ 253, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 20, 2 ] ], "id": "L20.mlp.3", "node": "l:326", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 321, 1 ] ], "source_columns_b": null, "target_columns": [ [ 254, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 22, 34 ] ], "id": "L20.mlp.4", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 44, 2 ], [ 51, 1 ], [ 216, 5 ], [ 222, 2 ], [ 236, 4 ], [ 256, 1 ], [ 262, 2 ] ], "source_columns_b": null, "target_columns": [ [ 44, 2 ], [ 51, 1 ], [ 216, 5 ], [ 222, 2 ], [ 236, 4 ], [ 256, 1 ], [ 262, 2 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 56, 34 ] ], "id": "L20.mlp.5", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 52, 4 ], [ 67, 1 ], [ 258, 4 ], [ 264, 3 ], [ 268, 3 ], [ 273, 1 ], [ 391, 1 ] ], "source_columns_b": null, "target_columns": [ [ 52, 4 ], [ 67, 1 ], [ 258, 4 ], [ 264, 3 ], [ 268, 3 ], [ 273, 1 ], [ 391, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 90, 34 ] ], "id": "L20.mlp.6", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 46, 1 ], [ 198, 9 ], [ 224, 7 ] ], "source_columns_b": null, "target_columns": [ [ 46, 1 ], [ 198, 9 ], [ 224, 7 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 124, 2 ] ], "id": "L20.mlp.7", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 40, 1 ] ], "source_columns_b": null, "target_columns": [ [ 40, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 126, 2 ] ], "id": "L20.mlp.8", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 321, 1 ] ], "source_columns_b": null, "target_columns": [ [ 321, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:69", "l:33", "l:275", "l:296", "l:274", "l:295", "l:326", "l:454", "l:472" ] }, { "active_attention_heads": 1, "active_mlp_neurons": 113, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L21.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": "l:32", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 139, 17 ], [ 28, 1 ], [ 121, 16 ], [ 39, 1 ], [ 49, 1 ], [ 194, 3 ], [ 240, 7 ], [ 248, 3 ], [ 252, 1 ], [ 335, 1 ] ], "source_columns_b": null, "target_columns": [ [ 40, 1 ], [ 44, 1 ], [ 198, 9 ], [ 216, 5 ], [ 222, 1 ] ], "type": "compute_linear" } ], "emitted_attention_heads": 1, "index": 21, "mlp_operations": [ { "hidden_slots": [ [ 1, 1 ] ], "id": "L21.mlp.0", "node": "l:31", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 40, 1 ], [ 44, 1 ], [ 198, 9 ], [ 216, 5 ], [ 222, 1 ] ], "source_columns_b": null, "target_columns": [ [ 223, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 2, 1 ] ], "id": "L21.mlp.1", "node": "l:383", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 57, 1 ], [ 274, 11 ], [ 288, 2 ], [ 302, 1 ], [ 337, 1 ], [ 339, 1 ] ], "source_columns_b": null, "target_columns": [ [ 224, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 3, 2 ] ], "id": "L21.mlp.2", "node": "l:273", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 50, 1 ] ], "source_columns_b": null, "target_columns": [ [ 225, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 5, 2 ] ], "id": "L21.mlp.3", "node": "l:294", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 253, 1 ] ], "source_columns_b": null, "target_columns": [ [ 226, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 7, 34 ] ], "id": "L21.mlp.4", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 139, 17 ] ], "source_columns_b": null, "target_columns": [ [ 139, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 41, 34 ] ], "id": "L21.mlp.5", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 28, 1 ], [ 121, 16 ] ], "source_columns_b": null, "target_columns": [ [ 28, 1 ], [ 121, 16 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 75, 34 ] ], "id": "L21.mlp.6", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 39, 1 ], [ 49, 1 ], [ 194, 3 ], [ 240, 7 ], [ 248, 3 ], [ 252, 1 ], [ 335, 1 ] ], "source_columns_b": null, "target_columns": [ [ 39, 1 ], [ 49, 1 ], [ 194, 3 ], [ 240, 7 ], [ 248, 3 ], [ 252, 1 ], [ 335, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 109, 2 ] ], "id": "L21.mlp.7", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 50, 1 ] ], "source_columns_b": null, "target_columns": [ [ 50, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 111, 2 ] ], "id": "L21.mlp.8", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 253, 1 ] ], "source_columns_b": null, "target_columns": [ [ 253, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:32", "l:31", "l:273", "l:294", "l:383", "l:433" ] }, { "active_attention_heads": 1, "active_mlp_neurons": 23, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L22.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": "l:272", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 223, 1 ] ], "source_columns_b": null, "target_columns": [ [ 121, 1 ] ], "type": "compute_linear" } ], "emitted_attention_heads": 1, "index": 22, "mlp_operations": [ { "hidden_slots": [ [ 1, 2 ] ], "id": "L22.mlp.0", "node": "l:270", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 121, 1 ], [ 225, 1 ] ], "source_columns_b": null, "target_columns": [ [ 122, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 3, 2 ] ], "id": "L22.mlp.1", "node": "l:292", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 121, 1 ], [ 225, 2 ] ], "source_columns_b": null, "target_columns": [ [ 123, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 5, 4 ] ], "id": "L22.mlp.2", "node": "l:30", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 223, 1 ] ], "source_columns_b": null, "target_columns": [ [ 124, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 9, 2 ] ], "id": "L22.mlp.3", "node": "l:324", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 121, 1 ], [ 225, 2 ], [ 254, 1 ] ], "source_columns_b": null, "target_columns": [ [ 125, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [ [ 11, 2 ] ], "id": "L22.mlp.4", "node": "l:381", "reuse_input_index": null, "role": "transport", "source_columns": [ [ 121, 1 ], [ 225, 2 ], [ 254, 1 ], [ 224, 1 ] ], "source_columns_b": null, "target_columns": [ [ 126, 1 ] ], "type": "compute_linear_bypass" }, { "hidden_slots": [], "id": "L22.mlp.5", "node": "l:272", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 121, 1 ] ], "type": "compute_bias" }, { "hidden_slots": [ [ 13, 2 ] ], "id": "L22.mlp.6", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 223, 1 ] ], "source_columns_b": null, "target_columns": [ [ 223, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 15, 2 ] ], "id": "L22.mlp.7", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 225, 1 ] ], "source_columns_b": null, "target_columns": [ [ 225, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 17, 2 ] ], "id": "L22.mlp.8", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 226, 1 ] ], "source_columns_b": null, "target_columns": [ [ 226, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 19, 2 ] ], "id": "L22.mlp.9", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 254, 1 ] ], "source_columns_b": null, "target_columns": [ [ 254, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 21, 2 ] ], "id": "L22.mlp.10", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 224, 1 ] ], "source_columns_b": null, "target_columns": [ [ 224, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:272", "l:271", "l:270", "l:293", "l:292", "l:325", "l:324", "l:382", "l:381", "l:30" ] }, { "active_attention_heads": 1, "active_mlp_neurons": 9, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L23.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 121, 1 ] ], "type": "cancel" } ], "emitted_attention_heads": 1, "index": 23, "mlp_operations": [ { "hidden_slots": [ [ 1, 2 ] ], "id": "L23.mlp.0", "node": "l:323", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 125, 1 ] ], "source_columns_b": null, "target_columns": [ [ 121, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 3, 2 ] ], "id": "L23.mlp.1", "node": "l:380", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 126, 1 ] ], "source_columns_b": null, "target_columns": [ [ 127, 1 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 5, 2 ] ], "id": "L23.mlp.2", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 125, 1 ] ], "source_columns_b": null, "target_columns": [ [ 125, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 7, 2 ] ], "id": "L23.mlp.3", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 126, 1 ] ], "source_columns_b": null, "target_columns": [ [ 126, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:323", "l:380", "l:29" ] }, { "active_attention_heads": 2, "active_mlp_neurons": 5, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L24.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": "l:28", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 124, 1 ], [ 122, 2 ], [ 121, 1 ], [ 127, 1 ] ], "source_columns_b": null, "target_columns": [ [ 121, 1 ] ], "type": "compute_linear" }, { "heads": [ 1, 1 ], "id": "L24.attn.1", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 121, 4 ] ], "type": "cancel" } ], "emitted_attention_heads": 2, "index": 24, "mlp_operations": [ { "hidden_slots": [ [ 1, 2 ] ], "id": "L24.mlp.0", "node": "l:27", "reuse_input_index": 0, "role": "transport", "source_columns": [ [ 364, 1 ] ], "source_columns_b": null, "target_columns": [ [ 121, 1 ] ], "type": "add_into_bypass" }, { "hidden_slots": [], "id": "L24.mlp.1", "node": "l:28", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 121, 1 ] ], "type": "compute_bias" }, { "hidden_slots": [ [ 3, 2 ] ], "id": "L24.mlp.2", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 127, 1 ] ], "source_columns_b": null, "target_columns": [ [ 127, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:28", "l:27" ] }, { "active_attention_heads": 1, "active_mlp_neurons": 47, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L25.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 364, 1 ] ], "type": "cancel" } ], "emitted_attention_heads": 1, "index": 25, "mlp_operations": [ { "hidden_slots": [ [ 1, 46 ] ], "id": "L25.mlp.0", "node": "l:26", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 121, 1 ] ], "source_columns_b": null, "target_columns": [ [ 122, 12 ] ], "type": "compute_ffn" }, { "hidden_slots": [], "id": "L25.mlp.1", "node": "l:423", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 28, 1 ], [ 134, 3 ], [ 139, 13 ] ], "type": "compute_literal_value" } ], "newly_computed": [ "l:423", "l:405", "l:25", "l:432", "l:453", "l:471", "l:481", "l:491", "l:501", "l:511", "l:26" ] }, { "active_attention_heads": 1, "active_mlp_neurons": 1963, "attention_operations": [ { "heads": [ 0, 1 ], "id": "L26.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 121, 1 ] ], "type": "cancel" } ], "emitted_attention_heads": 1, "index": 26, "mlp_operations": [ { "hidden_slots": [ [ 1, 204 ] ], "id": "L26.mlp.0", "node": "l:24", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 122, 12 ], [ 40, 1 ], [ 44, 1 ], [ 198, 9 ], [ 216, 5 ], [ 222, 1 ], [ 18, 1 ], [ 29, 4 ], [ 34, 1 ], [ 65, 1 ], [ 156, 8 ], [ 326, 1 ], [ 329, 1 ], [ 36, 3 ], [ 164, 9 ], [ 190, 4 ], [ 197, 1 ], [ 35, 1 ], [ 42, 2 ], [ 188, 1 ], [ 207, 9 ], [ 327, 1 ], [ 333, 2 ], [ 404, 1 ], [ 57, 1 ], [ 274, 11 ], [ 288, 2 ], [ 302, 1 ], [ 337, 1 ], [ 339, 1 ], [ 33, 1 ], [ 173, 15 ], [ 392, 1 ], [ 28, 1 ], [ 134, 3 ], [ 139, 13 ], [ 28, 1 ], [ 134, 3 ], [ 139, 13 ], [ 28, 1 ], [ 134, 3 ], [ 139, 13 ], [ 28, 1 ], [ 134, 3 ], [ 139, 13 ], [ 28, 1 ], [ 134, 3 ], [ 139, 13 ], [ 28, 1 ], [ 134, 3 ], [ 139, 13 ] ], "source_columns_b": null, "target_columns": [ [ 39, 1 ], [ 45, 2 ], [ 49, 8 ], [ 58, 7 ], [ 67, 2 ], [ 121, 1 ], [ 152, 4 ], [ 194, 3 ], [ 223, 4 ], [ 228, 3 ], [ 236, 11 ], [ 248, 3 ], [ 252, 2 ], [ 255, 2 ], [ 258, 6 ], [ 265, 2 ], [ 268, 3 ], [ 273, 1 ], [ 290, 12 ], [ 303, 1 ], [ 305, 2 ], [ 308, 4 ], [ 313, 1 ], [ 315, 2 ], [ 318, 1 ], [ 320, 3 ], [ 328, 1 ], [ 335, 2 ], [ 338, 1 ], [ 340, 5 ], [ 351, 1 ], [ 354, 1 ], [ 356, 27 ], [ 384, 5 ], [ 390, 2 ], [ 393, 2 ], [ 398, 6 ], [ 405, 9 ], [ 415, 51 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 205, 204 ] ], "id": "L26.mlp.1", "node": "l:431", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 122, 12 ], [ 18, 1 ], [ 29, 4 ], [ 34, 1 ], [ 65, 1 ], [ 156, 8 ], [ 326, 1 ], [ 329, 1 ], [ 36, 3 ], [ 164, 9 ], [ 190, 4 ], [ 197, 1 ], [ 35, 1 ], [ 42, 2 ], [ 188, 1 ], [ 207, 9 ], [ 327, 1 ], [ 333, 2 ], [ 404, 1 ], [ 57, 1 ], [ 274, 11 ], [ 288, 2 ], [ 302, 1 ], [ 337, 1 ], [ 339, 1 ], [ 33, 1 ], [ 173, 15 ], [ 392, 1 ], [ 27, 1 ], [ 69, 16 ], [ 40, 1 ], [ 44, 1 ], [ 198, 9 ], [ 216, 5 ], [ 222, 1 ], [ 18, 1 ], [ 29, 4 ], [ 34, 1 ], [ 65, 1 ], [ 156, 8 ], [ 326, 1 ], [ 329, 1 ], [ 36, 3 ], [ 164, 9 ], [ 190, 4 ], [ 197, 1 ], [ 35, 1 ], [ 42, 2 ], [ 188, 1 ], [ 207, 9 ], [ 327, 1 ], [ 333, 2 ], [ 404, 1 ], [ 57, 1 ], [ 274, 11 ], [ 288, 2 ], [ 302, 1 ], [ 337, 1 ], [ 339, 1 ], [ 33, 1 ], [ 173, 15 ], [ 392, 1 ] ], "source_columns_b": null, "target_columns": [ [ 19, 7 ], [ 102, 19 ], [ 345, 6 ], [ 352, 1 ], [ 395, 1 ], [ 466, 13 ], [ 480, 9 ], [ 490, 12 ], [ 505, 6 ], [ 512, 10 ], [ 525, 9 ], [ 535, 3 ], [ 540, 3 ], [ 544, 5 ], [ 550, 3 ], [ 555, 1 ], [ 557, 2 ], [ 560, 3 ], [ 565, 4 ], [ 570, 4 ], [ 576, 3 ], [ 580, 4 ], [ 585, 3 ], [ 601, 30 ], [ 632, 37 ], [ 670, 6 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 409, 204 ] ], "id": "L26.mlp.2", "node": "l:452", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 122, 12 ], [ 36, 3 ], [ 164, 9 ], [ 190, 4 ], [ 197, 1 ], [ 35, 1 ], [ 42, 2 ], [ 188, 1 ], [ 207, 9 ], [ 327, 1 ], [ 333, 2 ], [ 404, 1 ], [ 57, 1 ], [ 274, 11 ], [ 288, 2 ], [ 302, 1 ], [ 337, 1 ], [ 339, 1 ], [ 33, 1 ], [ 173, 15 ], [ 392, 1 ], [ 27, 1 ], [ 69, 33 ], [ 18, 1 ], [ 29, 4 ], [ 34, 1 ], [ 65, 1 ], [ 156, 8 ], [ 326, 1 ], [ 329, 1 ], [ 36, 3 ], [ 164, 9 ], [ 190, 4 ], [ 197, 1 ], [ 35, 1 ], [ 42, 2 ], [ 188, 1 ], [ 207, 9 ], [ 327, 1 ], [ 333, 2 ], [ 404, 1 ], [ 57, 1 ], [ 274, 11 ], [ 288, 2 ], [ 302, 1 ], [ 337, 1 ], [ 339, 1 ], [ 33, 1 ], [ 173, 15 ], [ 392, 1 ], [ 27, 1 ], [ 69, 16 ] ], "source_columns_b": null, "target_columns": [ [ 66, 1 ], [ 137, 2 ], [ 331, 2 ], [ 353, 1 ], [ 355, 1 ], [ 396, 2 ], [ 676, 3 ], [ 680, 6 ], [ 691, 44 ], [ 736, 68 ], [ 805, 16 ], [ 822, 33 ], [ 864, 25 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 613, 204 ] ], "id": "L26.mlp.3", "node": "l:470", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 122, 12 ], [ 35, 1 ], [ 42, 2 ], [ 188, 1 ], [ 207, 9 ], [ 327, 1 ], [ 333, 2 ], [ 404, 1 ], [ 57, 1 ], [ 274, 11 ], [ 288, 2 ], [ 302, 1 ], [ 337, 1 ], [ 339, 1 ], [ 33, 1 ], [ 173, 15 ], [ 392, 1 ], [ 27, 1 ], [ 69, 33 ], [ 85, 17 ], [ 36, 3 ], [ 164, 9 ], [ 190, 4 ], [ 197, 1 ], [ 35, 1 ], [ 42, 2 ], [ 188, 1 ], [ 207, 9 ], [ 327, 1 ], [ 333, 2 ], [ 404, 1 ], [ 57, 1 ], [ 274, 11 ], [ 288, 2 ], [ 302, 1 ], [ 337, 1 ], [ 339, 1 ], [ 33, 1 ], [ 173, 15 ], [ 392, 1 ], [ 27, 1 ], [ 69, 33 ] ], "source_columns_b": null, "target_columns": [ [ 889, 204 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 817, 204 ] ], "id": "L26.mlp.4", "node": "l:480", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 122, 12 ], [ 57, 1 ], [ 274, 11 ], [ 288, 2 ], [ 302, 1 ], [ 337, 1 ], [ 339, 1 ], [ 33, 1 ], [ 173, 15 ], [ 392, 1 ], [ 27, 1 ], [ 69, 33 ], [ 85, 17 ], [ 85, 17 ], [ 35, 1 ], [ 42, 2 ], [ 188, 1 ], [ 207, 9 ], [ 327, 1 ], [ 333, 2 ], [ 404, 1 ], [ 57, 1 ], [ 274, 11 ], [ 288, 2 ], [ 302, 1 ], [ 337, 1 ], [ 339, 1 ], [ 33, 1 ], [ 173, 15 ], [ 392, 1 ], [ 27, 1 ], [ 69, 33 ], [ 85, 17 ] ], "source_columns_b": null, "target_columns": [ [ 1093, 204 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 1021, 204 ] ], "id": "L26.mlp.5", "node": "l:490", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 122, 12 ], [ 33, 1 ], [ 173, 15 ], [ 392, 1 ], [ 27, 1 ], [ 69, 33 ], [ 85, 17 ], [ 85, 17 ], [ 85, 17 ], [ 57, 1 ], [ 274, 11 ], [ 288, 2 ], [ 302, 1 ], [ 337, 1 ], [ 339, 1 ], [ 33, 1 ], [ 173, 15 ], [ 392, 1 ], [ 27, 1 ], [ 69, 33 ], [ 85, 17 ], [ 85, 17 ] ], "source_columns_b": null, "target_columns": [ [ 1297, 204 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 1225, 204 ] ], "id": "L26.mlp.6", "node": "l:500", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 122, 12 ], [ 27, 1 ], [ 69, 33 ], [ 85, 17 ], [ 85, 17 ], [ 85, 17 ], [ 85, 17 ], [ 33, 1 ], [ 173, 15 ], [ 392, 1 ], [ 27, 1 ], [ 69, 33 ], [ 85, 17 ], [ 85, 17 ], [ 85, 17 ] ], "source_columns_b": null, "target_columns": [ [ 1501, 204 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 1429, 204 ] ], "id": "L26.mlp.7", "node": "l:510", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 122, 12 ], [ 85, 17 ], [ 85, 17 ], [ 85, 17 ], [ 85, 17 ], [ 85, 17 ], [ 85, 17 ], [ 27, 1 ], [ 69, 33 ], [ 85, 17 ], [ 85, 17 ], [ 85, 17 ], [ 85, 17 ] ], "source_columns_b": null, "target_columns": [ [ 1705, 204 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 1633, 34 ] ], "id": "L26.mlp.8", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 40, 1 ], [ 44, 1 ], [ 198, 9 ], [ 216, 5 ], [ 222, 1 ] ], "source_columns_b": null, "target_columns": [ [ 40, 1 ], [ 44, 1 ], [ 198, 9 ], [ 216, 5 ], [ 222, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1667, 34 ] ], "id": "L26.mlp.9", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 18, 1 ], [ 29, 4 ], [ 34, 1 ], [ 65, 1 ], [ 156, 8 ], [ 326, 1 ], [ 329, 1 ] ], "source_columns_b": null, "target_columns": [ [ 18, 1 ], [ 29, 4 ], [ 34, 1 ], [ 65, 1 ], [ 156, 8 ], [ 326, 1 ], [ 329, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1701, 34 ] ], "id": "L26.mlp.10", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 36, 3 ], [ 164, 9 ], [ 190, 4 ], [ 197, 1 ] ], "source_columns_b": null, "target_columns": [ [ 36, 3 ], [ 164, 9 ], [ 190, 4 ], [ 197, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1735, 34 ] ], "id": "L26.mlp.11", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 35, 1 ], [ 42, 2 ], [ 188, 1 ], [ 207, 9 ], [ 327, 1 ], [ 333, 2 ], [ 404, 1 ] ], "source_columns_b": null, "target_columns": [ [ 35, 1 ], [ 42, 2 ], [ 188, 1 ], [ 207, 9 ], [ 327, 1 ], [ 333, 2 ], [ 404, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1769, 34 ] ], "id": "L26.mlp.12", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 57, 1 ], [ 274, 11 ], [ 288, 2 ], [ 302, 1 ], [ 337, 1 ], [ 339, 1 ] ], "source_columns_b": null, "target_columns": [ [ 57, 1 ], [ 274, 11 ], [ 288, 2 ], [ 302, 1 ], [ 337, 1 ], [ 339, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1803, 34 ] ], "id": "L26.mlp.13", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 33, 1 ], [ 173, 15 ], [ 392, 1 ] ], "source_columns_b": null, "target_columns": [ [ 33, 1 ], [ 173, 15 ], [ 392, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1837, 34 ] ], "id": "L26.mlp.14", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 27, 1 ], [ 69, 16 ] ], "source_columns_b": null, "target_columns": [ [ 27, 1 ], [ 69, 16 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1871, 34 ] ], "id": "L26.mlp.15", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 85, 17 ] ], "source_columns_b": null, "target_columns": [ [ 85, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1905, 34 ] ], "id": "L26.mlp.16", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 28, 1 ], [ 134, 3 ], [ 139, 13 ] ], "source_columns_b": null, "target_columns": [ [ 28, 1 ], [ 134, 3 ], [ 139, 13 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1939, 24 ] ], "id": "L26.mlp.17", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 122, 12 ] ], "source_columns_b": null, "target_columns": [ [ 122, 12 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:24", "l:431", "l:452", "l:470", "l:480", "l:490", "l:500", "l:510" ] }, { "active_attention_heads": 32, "active_mlp_neurons": 3415, "attention_operations": [ { "heads": [ 0, 4 ], "id": "L27.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": "l:23", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 39, 1 ], [ 45, 2 ], [ 49, 8 ], [ 58, 7 ], [ 67, 2 ], [ 121, 1 ], [ 152, 4 ], [ 194, 3 ], [ 223, 4 ], [ 228, 3 ], [ 236, 11 ], [ 248, 3 ], [ 252, 2 ], [ 255, 2 ], [ 258, 6 ], [ 265, 2 ], [ 268, 3 ], [ 273, 1 ], [ 290, 12 ], [ 303, 1 ], [ 305, 2 ], [ 308, 4 ], [ 313, 1 ], [ 315, 2 ], [ 318, 1 ], [ 320, 3 ], [ 328, 1 ], [ 335, 2 ], [ 338, 1 ], [ 340, 5 ], [ 351, 1 ], [ 354, 1 ], [ 356, 27 ], [ 384, 5 ], [ 390, 2 ], [ 393, 2 ], [ 398, 6 ], [ 405, 9 ], [ 415, 51 ] ], "source_columns_b": null, "target_columns": [ [ 18, 1 ], [ 65, 1 ], [ 81, 13 ], [ 326, 1 ], [ 329, 1 ] ], "type": "compute_linear" }, { "heads": [ 4, 4 ], "id": "L27.attn.1", "key_source_columns": [ [ 0, 1 ] ], "node": "l:430", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 19, 7 ], [ 102, 19 ], [ 345, 6 ], [ 352, 1 ], [ 395, 1 ], [ 466, 13 ], [ 480, 9 ], [ 490, 12 ], [ 505, 6 ], [ 512, 10 ], [ 525, 9 ], [ 535, 3 ], [ 540, 3 ], [ 544, 5 ], [ 550, 3 ], [ 555, 1 ], [ 557, 2 ], [ 560, 3 ], [ 565, 4 ], [ 570, 4 ], [ 576, 3 ], [ 580, 4 ], [ 585, 3 ], [ 601, 30 ], [ 632, 37 ], [ 670, 6 ] ], "source_columns_b": null, "target_columns": [ [ 69, 2 ], [ 79, 1 ], [ 94, 8 ], [ 122, 6 ] ], "type": "compute_linear" }, { "heads": [ 8, 4 ], "id": "L27.attn.2", "key_source_columns": [ [ 0, 1 ] ], "node": "l:451", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 66, 1 ], [ 137, 2 ], [ 331, 2 ], [ 353, 1 ], [ 355, 1 ], [ 396, 2 ], [ 676, 3 ], [ 680, 6 ], [ 691, 44 ], [ 736, 68 ], [ 805, 16 ], [ 822, 33 ], [ 864, 25 ] ], "source_columns_b": null, "target_columns": [ [ 27, 2 ], [ 128, 9 ], [ 139, 6 ] ], "type": "compute_linear" }, { "heads": [ 12, 4 ], "id": "L27.attn.3", "key_source_columns": [ [ 0, 1 ] ], "node": "l:469", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 889, 204 ] ], "source_columns_b": null, "target_columns": [ [ 29, 3 ], [ 71, 1 ], [ 145, 7 ], [ 156, 6 ] ], "type": "compute_linear" }, { "heads": [ 16, 4 ], "id": "L27.attn.4", "key_source_columns": [ [ 0, 1 ] ], "node": "l:479", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1093, 204 ] ], "source_columns_b": null, "target_columns": [ [ 32, 3 ], [ 162, 14 ] ], "type": "compute_linear" }, { "heads": [ 20, 4 ], "id": "L27.attn.5", "key_source_columns": [ [ 0, 1 ] ], "node": "l:489", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1297, 204 ] ], "source_columns_b": null, "target_columns": [ [ 35, 3 ], [ 72, 1 ], [ 176, 12 ], [ 392, 1 ] ], "type": "compute_linear" }, { "heads": [ 24, 4 ], "id": "L27.attn.6", "key_source_columns": [ [ 0, 1 ] ], "node": "l:499", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1501, 204 ] ], "source_columns_b": null, "target_columns": [ [ 38, 1 ], [ 40, 1 ], [ 73, 1 ], [ 188, 1 ], [ 190, 4 ], [ 197, 6 ], [ 327, 1 ], [ 333, 1 ], [ 404, 1 ] ], "type": "compute_linear" }, { "heads": [ 28, 4 ], "id": "L27.attn.7", "key_source_columns": [ [ 0, 1 ] ], "node": "l:509", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 1705, 204 ] ], "source_columns_b": null, "target_columns": [ [ 42, 2 ], [ 203, 14 ], [ 334, 1 ] ], "type": "compute_linear" } ], "emitted_attention_heads": 32, "index": 27, "mlp_operations": [ { "hidden_slots": [ [ 1, 17 ] ], "id": "L27.mlp.0", "node": "l:12", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 285, 1 ], [ 18, 1 ], [ 65, 1 ], [ 81, 13 ], [ 326, 1 ], [ 329, 1 ] ], "source_columns_b": null, "target_columns": [ [ 44, 1 ], [ 74, 3 ], [ 80, 1 ], [ 217, 4 ], [ 222, 1 ], [ 274, 7 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 18, 17 ] ], "id": "L27.mlp.1", "node": "l:424", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 286, 1 ], [ 69, 2 ], [ 79, 1 ], [ 94, 8 ], [ 122, 6 ] ], "source_columns_b": null, "target_columns": [ [ 57, 1 ], [ 77, 2 ], [ 281, 4 ], [ 288, 2 ], [ 302, 1 ], [ 337, 1 ], [ 339, 1 ], [ 1909, 5 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 35, 17 ] ], "id": "L27.mlp.2", "node": "l:445", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 231, 1 ], [ 27, 2 ], [ 128, 9 ], [ 139, 6 ] ], "source_columns_b": null, "target_columns": [ [ 1914, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 52, 17 ] ], "id": "L27.mlp.3", "node": "l:463", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 232, 1 ], [ 29, 3 ], [ 71, 1 ], [ 145, 7 ], [ 156, 6 ] ], "source_columns_b": null, "target_columns": [ [ 1931, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 69, 17 ] ], "id": "L27.mlp.4", "node": "l:473", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 233, 1 ], [ 32, 3 ], [ 162, 14 ] ], "source_columns_b": null, "target_columns": [ [ 1948, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 86, 17 ] ], "id": "L27.mlp.5", "node": "l:483", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 287, 1 ], [ 35, 3 ], [ 72, 1 ], [ 176, 12 ], [ 392, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1965, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 103, 17 ] ], "id": "L27.mlp.6", "node": "l:493", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 234, 1 ], [ 38, 1 ], [ 40, 1 ], [ 73, 1 ], [ 188, 1 ], [ 190, 4 ], [ 197, 6 ], [ 327, 1 ], [ 333, 1 ], [ 404, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1982, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 120, 17 ] ], "id": "L27.mlp.7", "node": "l:503", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 235, 1 ], [ 42, 2 ], [ 203, 14 ], [ 334, 1 ] ], "source_columns_b": null, "target_columns": [ [ 1999, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [], "id": "L27.mlp.8", "node": "l:513", "reuse_input_index": null, "role": "transport", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 2016, 17 ] ], "type": "compute_literal_value" }, { "hidden_slots": [ [ 137, 408 ] ], "id": "L27.mlp.9", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 39, 1 ], [ 45, 2 ], [ 49, 8 ], [ 58, 7 ], [ 67, 2 ], [ 121, 1 ], [ 152, 4 ], [ 194, 3 ], [ 223, 4 ], [ 228, 3 ], [ 236, 11 ], [ 248, 3 ], [ 252, 2 ], [ 255, 2 ], [ 258, 6 ], [ 265, 2 ], [ 268, 3 ], [ 273, 1 ], [ 290, 12 ], [ 303, 1 ], [ 305, 2 ], [ 308, 4 ], [ 313, 1 ], [ 315, 2 ], [ 318, 1 ], [ 320, 3 ], [ 328, 1 ], [ 335, 2 ], [ 338, 1 ], [ 340, 5 ], [ 351, 1 ], [ 354, 1 ], [ 356, 27 ], [ 384, 5 ], [ 390, 2 ], [ 393, 2 ], [ 398, 6 ], [ 405, 9 ], [ 415, 51 ] ], "source_columns_b": null, "target_columns": [ [ 39, 1 ], [ 45, 2 ], [ 49, 8 ], [ 58, 7 ], [ 67, 2 ], [ 121, 1 ], [ 152, 4 ], [ 194, 3 ], [ 223, 4 ], [ 228, 3 ], [ 236, 11 ], [ 248, 3 ], [ 252, 2 ], [ 255, 2 ], [ 258, 6 ], [ 265, 2 ], [ 268, 3 ], [ 273, 1 ], [ 290, 12 ], [ 303, 1 ], [ 305, 2 ], [ 308, 4 ], [ 313, 1 ], [ 315, 2 ], [ 318, 1 ], [ 320, 3 ], [ 328, 1 ], [ 335, 2 ], [ 338, 1 ], [ 340, 5 ], [ 351, 1 ], [ 354, 1 ], [ 356, 27 ], [ 384, 5 ], [ 390, 2 ], [ 393, 2 ], [ 398, 6 ], [ 405, 9 ], [ 415, 51 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 545, 408 ] ], "id": "L27.mlp.10", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 19, 7 ], [ 102, 19 ], [ 345, 6 ], [ 352, 1 ], [ 395, 1 ], [ 466, 13 ], [ 480, 9 ], [ 490, 12 ], [ 505, 6 ], [ 512, 10 ], [ 525, 9 ], [ 535, 3 ], [ 540, 3 ], [ 544, 5 ], [ 550, 3 ], [ 555, 1 ], [ 557, 2 ], [ 560, 3 ], [ 565, 4 ], [ 570, 4 ], [ 576, 3 ], [ 580, 4 ], [ 585, 3 ], [ 601, 30 ], [ 632, 37 ], [ 670, 6 ] ], "source_columns_b": null, "target_columns": [ [ 19, 7 ], [ 102, 19 ], [ 345, 6 ], [ 352, 1 ], [ 395, 1 ], [ 466, 13 ], [ 480, 9 ], [ 490, 12 ], [ 505, 6 ], [ 512, 10 ], [ 525, 9 ], [ 535, 3 ], [ 540, 3 ], [ 544, 5 ], [ 550, 3 ], [ 555, 1 ], [ 557, 2 ], [ 560, 3 ], [ 565, 4 ], [ 570, 4 ], [ 576, 3 ], [ 580, 4 ], [ 585, 3 ], [ 601, 30 ], [ 632, 37 ], [ 670, 6 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 953, 408 ] ], "id": "L27.mlp.11", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 66, 1 ], [ 137, 2 ], [ 331, 2 ], [ 353, 1 ], [ 355, 1 ], [ 396, 2 ], [ 676, 3 ], [ 680, 6 ], [ 691, 44 ], [ 736, 68 ], [ 805, 16 ], [ 822, 33 ], [ 864, 25 ] ], "source_columns_b": null, "target_columns": [ [ 66, 1 ], [ 137, 2 ], [ 331, 2 ], [ 353, 1 ], [ 355, 1 ], [ 396, 2 ], [ 676, 3 ], [ 680, 6 ], [ 691, 44 ], [ 736, 68 ], [ 805, 16 ], [ 822, 33 ], [ 864, 25 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1361, 408 ] ], "id": "L27.mlp.12", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 889, 204 ] ], "source_columns_b": null, "target_columns": [ [ 889, 204 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 1769, 408 ] ], "id": "L27.mlp.13", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1093, 204 ] ], "source_columns_b": null, "target_columns": [ [ 1093, 204 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 2177, 408 ] ], "id": "L27.mlp.14", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1297, 204 ] ], "source_columns_b": null, "target_columns": [ [ 1297, 204 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 2585, 408 ] ], "id": "L27.mlp.15", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1501, 204 ] ], "source_columns_b": null, "target_columns": [ [ 1501, 204 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 2993, 408 ] ], "id": "L27.mlp.16", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1705, 204 ] ], "source_columns_b": null, "target_columns": [ [ 1705, 204 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 3401, 2 ] ], "id": "L27.mlp.17", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 285, 1 ] ], "source_columns_b": null, "target_columns": [ [ 285, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 3403, 2 ] ], "id": "L27.mlp.18", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 286, 1 ] ], "source_columns_b": null, "target_columns": [ [ 286, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 3405, 2 ] ], "id": "L27.mlp.19", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 231, 1 ] ], "source_columns_b": null, "target_columns": [ [ 231, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 3407, 2 ] ], "id": "L27.mlp.20", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 233, 1 ] ], "source_columns_b": null, "target_columns": [ [ 233, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 3409, 2 ] ], "id": "L27.mlp.21", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 287, 1 ] ], "source_columns_b": null, "target_columns": [ [ 287, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 3411, 2 ] ], "id": "L27.mlp.22", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 234, 1 ] ], "source_columns_b": null, "target_columns": [ [ 234, 1 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 3413, 2 ] ], "id": "L27.mlp.23", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 235, 1 ] ], "source_columns_b": null, "target_columns": [ [ 235, 1 ] ], "type": "cancel_bypass" } ], "newly_computed": [ "l:23", "l:430", "l:451", "l:469", "l:479", "l:489", "l:499", "l:509", "l:13", "l:12", "l:425", "l:424", "l:446", "l:445", "l:464", "l:463", "l:474", "l:473", "l:484", "l:483", "l:494", "l:493", "l:504", "l:503", "l:11", "l:513" ] }, { "active_attention_heads": 6, "active_mlp_neurons": 307, "attention_operations": [ { "heads": [ 0, 3 ], "id": "L28.attn.0", "key_source_columns": [ [ 0, 1 ] ], "node": "l:10", "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "transport", "source_columns": [ [ 44, 1 ], [ 74, 3 ], [ 80, 1 ], [ 217, 4 ], [ 222, 1 ], [ 274, 7 ], [ 57, 1 ], [ 77, 2 ], [ 281, 4 ], [ 288, 2 ], [ 302, 1 ], [ 337, 1 ], [ 339, 1 ], [ 1909, 107 ] ], "source_columns_b": null, "target_columns": [ [ 18, 1 ], [ 27, 1 ], [ 65, 1 ], [ 82, 8 ], [ 91, 2 ], [ 329, 1 ], [ 1631, 1 ], [ 1645, 2 ] ], "type": "compute_linear" }, { "heads": [ 3, 3 ], "id": "L28.attn.1", "key_source_columns": [ [ 0, 1 ] ], "node": null, "query_source_columns": [ [ 0, 1 ] ], "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 18, 1 ], [ 27, 2 ], [ 32, 7 ], [ 40, 1 ], [ 42, 2 ], [ 65, 1 ], [ 69, 2 ], [ 72, 2 ], [ 79, 1 ], [ 81, 21 ], [ 122, 15 ], [ 139, 6 ], [ 162, 27 ], [ 190, 4 ], [ 197, 20 ], [ 232, 1 ], [ 326, 2 ], [ 329, 1 ], [ 333, 2 ], [ 392, 1 ], [ 404, 1 ], [ 1931, 17 ] ], "type": "cancel" } ], "emitted_attention_heads": 6, "index": 28, "mlp_operations": [ { "hidden_slots": [ [ 1, 34 ] ], "id": "L28.mlp.0", "node": "l:0", "reuse_input_index": null, "role": "semantic_compute", "source_columns": [ [ 26, 1 ], [ 18, 1 ], [ 27, 1 ], [ 65, 1 ], [ 82, 8 ], [ 91, 2 ], [ 329, 1 ], [ 1631, 1 ], [ 1645, 2 ], [ 2016, 17 ] ], "source_columns_b": null, "target_columns": [ [ 1, 17 ] ], "type": "compute_ffn" }, { "hidden_slots": [ [ 35, 34 ] ], "id": "L28.mlp.1", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 29, 3 ], [ 71, 1 ], [ 145, 7 ], [ 156, 6 ] ], "source_columns_b": null, "target_columns": [ [ 29, 3 ], [ 71, 1 ], [ 145, 7 ], [ 156, 6 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 69, 34 ] ], "id": "L28.mlp.2", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 44, 1 ], [ 74, 3 ], [ 80, 1 ], [ 217, 4 ], [ 222, 1 ], [ 274, 7 ] ], "source_columns_b": null, "target_columns": [ [ 44, 1 ], [ 74, 3 ], [ 80, 1 ], [ 217, 4 ], [ 222, 1 ], [ 274, 7 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 103, 34 ] ], "id": "L28.mlp.3", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 57, 1 ], [ 77, 2 ], [ 281, 4 ], [ 288, 2 ], [ 302, 1 ], [ 337, 1 ], [ 339, 1 ], [ 1909, 5 ] ], "source_columns_b": null, "target_columns": [ [ 57, 1 ], [ 77, 2 ], [ 281, 4 ], [ 288, 2 ], [ 302, 1 ], [ 337, 1 ], [ 339, 1 ], [ 1909, 5 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 137, 34 ] ], "id": "L28.mlp.4", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1914, 17 ] ], "source_columns_b": null, "target_columns": [ [ 1914, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 171, 34 ] ], "id": "L28.mlp.5", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1948, 17 ] ], "source_columns_b": null, "target_columns": [ [ 1948, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 205, 34 ] ], "id": "L28.mlp.6", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1965, 17 ] ], "source_columns_b": null, "target_columns": [ [ 1965, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 239, 34 ] ], "id": "L28.mlp.7", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1982, 17 ] ], "source_columns_b": null, "target_columns": [ [ 1982, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [ [ 273, 34 ] ], "id": "L28.mlp.8", "node": null, "reuse_input_index": null, "role": "memory_management", "source_columns": [ [ 1999, 17 ] ], "source_columns_b": null, "target_columns": [ [ 1999, 17 ] ], "type": "cancel_bypass" }, { "hidden_slots": [], "id": "L28.mlp.9", "node": "i:0", "reuse_input_index": null, "role": "memory_management", "source_columns": null, "source_columns_b": null, "target_columns": [ [ 0, 1 ] ], "type": "clear_literal_seed" } ], "newly_computed": [ "l:10", "l:1", "l:0" ] } ], "realizations": [ { "candidates": [ "mlp_composite" ], "node": "l:46", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:48", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:50", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:45", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:47", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:49", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:43", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:7", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:54", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:55", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:57", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:52", "selected": "attn_heads" }, { "candidates": [ "mlp_composite" ], "node": "l:51", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:41", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:38", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:39", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:58", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:36", "selected": "attn_heads" }, { "candidates": [ "mlp_literal" ], "node": "l:64", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:65", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:67", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:62", "selected": "attn_heads" }, { "candidates": [ "mlp_literal" ], "node": "l:73", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:74", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:76", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:71", "selected": "attn_heads" }, { "candidates": [ "mlp_composite" ], "node": "l:106", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:119", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:120", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:122", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:117", "selected": "attn_heads" }, { "candidates": [ "mlp_composite" ], "node": "l:115", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:116", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:113", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:166", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:164", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:103", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:104", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:107", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:101", "selected": "attn_heads" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:108", "selected": "mlp_bypass" }, { "candidates": [ "mlp_literal" ], "node": "l:100", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:99", "selected": "attn_heads" }, { "candidates": [ "mlp_composite" ], "node": "l:98", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:97", "selected": "attn_transport" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:126", "selected": "mlp_bypass" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:127", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:124", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:123", "selected": "mlp_bypass" }, { "candidates": [ "mlp_composite" ], "node": "l:111", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:128", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:131", "selected": "mlp_literal" }, { "candidates": [ "mlp_composite" ], "node": "l:129", "selected": "mlp_composite" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:109", "selected": "attn_add" }, { "candidates": [ "mlp_literal" ], "node": "l:134", "selected": "mlp_literal" }, { "candidates": [ "mlp_composite" ], "node": "l:132", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:155", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:156", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:158", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:153", "selected": "attn_heads" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:159", "selected": "mlp_bypass" }, { "candidates": [ "mlp_literal" ], "node": "l:152", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:151", "selected": "attn_heads" }, { "candidates": [ "mlp_composite" ], "node": "l:150", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:149", "selected": "attn_transport" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:170", "selected": "mlp_bypass" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:171", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:168", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:167", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:162", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:172", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:175", "selected": "mlp_literal" }, { "candidates": [ "mlp_composite" ], "node": "l:173", "selected": "mlp_composite" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:160", "selected": "attn_add" }, { "candidates": [ "mlp_literal" ], "node": "l:178", "selected": "mlp_literal" }, { "candidates": [ "mlp_composite" ], "node": "l:176", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:93", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:94", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:96", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:91", "selected": "attn_heads" }, { "candidates": [ "mlp_literal" ], "node": "l:145", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:146", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:148", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:143", "selected": "attn_heads" }, { "candidates": [ "attn_heads" ], "node": "l:89", "selected": "attn_heads" }, { "candidates": [ "mlp_literal" ], "node": "l:85", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:86", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:88", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:83", "selected": "attn_heads" }, { "candidates": [ "attn_heads" ], "node": "l:200", "selected": "attn_heads" }, { "candidates": [ "mlp_literal" ], "node": "l:196", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:197", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:199", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:194", "selected": "attn_heads" }, { "candidates": [ "attn_heads" ], "node": "l:220", "selected": "attn_heads" }, { "candidates": [ "mlp_literal" ], "node": "l:216", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:217", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:219", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:214", "selected": "attn_heads" }, { "candidates": [ "attn_heads" ], "node": "l:141", "selected": "attn_heads" }, { "candidates": [ "mlp_literal" ], "node": "l:137", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:138", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:140", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:135", "selected": "attn_heads" }, { "candidates": [ "attn_heads" ], "node": "l:190", "selected": "attn_heads" }, { "candidates": [ "mlp_literal" ], "node": "l:186", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:187", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:189", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:184", "selected": "attn_heads" }, { "candidates": [ "attn_heads" ], "node": "l:228", "selected": "attn_heads" }, { "candidates": [ "mlp_literal" ], "node": "l:224", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:225", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:227", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:222", "selected": "attn_heads" }, { "candidates": [ "mlp_literal" ], "node": "l:302", "selected": "mlp_literal" }, { "candidates": [ "mlp_composite" ], "node": "l:306", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:310", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:312", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:335", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:308", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:304", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:412", "selected": "mlp_literal" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:390", "selected": "mlp_bypass" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:334", "selected": "mlp_bypass" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:303", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:410", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:388", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:332", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:300", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:59", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:438", "selected": "mlp_literal" }, { "candidates": [ "mlp_composite" ], "node": "l:347", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:351", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:355", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:346", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:349", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:350", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:353", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:354", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:357", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:344", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:343", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:341", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:371", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:365", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:358", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:373", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:367", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:363", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:369", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:361", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:417", "selected": "mlp_literal" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:395", "selected": "attn_transport" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:360", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:415", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:393", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:339", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:68", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:441", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:231", "selected": "mlp_literal" }, { "candidates": [ "mlp_composite" ], "node": "l:81", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:182", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:251", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:192", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:253", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:241", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:255", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:243", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:212", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:211", "selected": "attn_transport" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:230", "selected": "mlp_bypass" }, { "candidates": [ "mlp_composite" ], "node": "l:209", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:208", "selected": "mlp_bypass" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:207", "selected": "attn_transport" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:245", "selected": "attn_transport" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:239", "selected": "mlp_bypass" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:249", "selected": "attn_transport" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:263", "selected": "attn_transport" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:180", "selected": "attn_transport" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:80", "selected": "mlp_bypass" }, { "candidates": [ "mlp_composite" ], "node": "l:205", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:236", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:246", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:260", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:266", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:288", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:234", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:290", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:258", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:321", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:286", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:232", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:203", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:422", "selected": "mlp_literal" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:400", "selected": "attn_transport" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:379", "selected": "attn_transport" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:320", "selected": "attn_transport" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:285", "selected": "attn_transport" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:202", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:420", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:398", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:377", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:318", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:283", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:77", "selected": "mlp_composite" }, { "candidates": [ "mlp_literal" ], "node": "l:444", "selected": "mlp_literal" }, { "candidates": [ "mlp_composite" ], "node": "l:34", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:60", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:69", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:32", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:277", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:279", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:281", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:275", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:298", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:314", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:316", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:296", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:330", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:337", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:375", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:328", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:386", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:391", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:396", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:384", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:408", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:413", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:418", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:406", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:436", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:439", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:442", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:434", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:457", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:459", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:461", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:455", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:404", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:402", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:31", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:272", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:274", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:273", "selected": "mlp_bypass" }, { "candidates": [ "mlp_composite" ], "node": "l:295", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:294", "selected": "mlp_bypass" }, { "candidates": [ "mlp_composite" ], "node": "l:327", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:326", "selected": "mlp_bypass" }, { "candidates": [ "mlp_composite" ], "node": "l:383", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:270", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:292", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:324", "selected": "mlp_bypass" }, { "candidates": [ "mlp_composite" ], "node": "l:323", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:381", "selected": "mlp_bypass" }, { "candidates": [ "mlp_composite" ], "node": "l:380", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:28", "selected": "attn_transport" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:401", "selected": "attn_transport" }, { "candidates": [ "attn_add", "mlp_add" ], "node": "l:27", "selected": "mlp_add" }, { "candidates": [ "mlp_literal" ], "node": "l:423", "selected": "mlp_literal" }, { "candidates": [ "mlp_composite" ], "node": "l:24", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:23", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:431", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:430", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:452", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:451", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:470", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:469", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:480", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:479", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:490", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:489", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:500", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:499", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:510", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:509", "selected": "attn_transport" }, { "candidates": [ "mlp_literal" ], "node": "l:4", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:5", "selected": "mlp_literal" }, { "candidates": [ "mlp_literal" ], "node": "l:9", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:2", "selected": "attn_heads" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:21", "selected": "mlp_bypass" }, { "candidates": [ "mlp_literal" ], "node": "l:20", "selected": "mlp_literal" }, { "candidates": [ "attn_heads" ], "node": "l:19", "selected": "attn_heads" }, { "candidates": [ "mlp_composite" ], "node": "l:18", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:17", "selected": "attn_transport" }, { "candidates": [ "mlp_composite" ], "node": "l:16", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:22", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:14", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:12", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:428", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:429", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:426", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:424", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:449", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:450", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:447", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:445", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:467", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:468", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:465", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:463", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:477", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:478", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:475", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:473", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:487", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:488", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:485", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:483", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:497", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:498", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:495", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:493", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:507", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:508", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:505", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:503", "selected": "mlp_composite" }, { "candidates": [ "attn_transport", "mlp_bypass" ], "node": "l:10", "selected": "attn_transport" }, { "candidates": [ "mlp_literal" ], "node": "l:513", "selected": "mlp_literal" }, { "candidates": [ "mlp_composite" ], "node": "l:0", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:229", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:191", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:142", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:221", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:201", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:90", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:238", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:257", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:248", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:265", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:262", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:269", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:268", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:179", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:79", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:30", "selected": "mlp_composite" }, { "candidates": [ "mlp_composite" ], "node": "l:26", "selected": "mlp_composite" } ] }, "token_io": { "bos_token_id": 15, "embedding_residual_columns": [ [ 1, 17 ] ], "eos_token_id": 16, "output_residual_columns": [ [ 1, 17 ] ], "readout": { "checkpoint_tensor": "model.embed_tokens.weight", "final_normalization": "rms_norm", "formula": "logits[token] = final_normalize(residual) @ embedding[token]", "input_state": "output", "kind": "tied_embedding_dot_product" }, "source_embedding_node": "s:8", "unknown_token_id": 17, "vocabulary_sha256": "907a098ae3a8687a22cd33300ec72d9bb9d997acddefeec8d226522dc818ee2c", "vocabulary_size": 18 } }