{
  "schemaVersion": 1,
  "modelFamily": "ming_3_tts_tiny_language",
  "alignment": 16384,
  "file": "weights.bin",
  "quantizedLinears": {
    "model.model.layers.1.mlp.experts.down_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.1.mlp.experts.gate_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.1.mlp.experts.up_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.10.mlp.experts.down_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.10.mlp.experts.gate_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.10.mlp.experts.up_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.11.mlp.experts.down_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.11.mlp.experts.gate_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.11.mlp.experts.up_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.12.mlp.experts.down_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.12.mlp.experts.gate_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.12.mlp.experts.up_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.13.mlp.experts.down_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.13.mlp.experts.gate_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.13.mlp.experts.up_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.14.mlp.experts.down_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.14.mlp.experts.gate_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.14.mlp.experts.up_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.15.mlp.experts.down_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.15.mlp.experts.gate_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.15.mlp.experts.up_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.16.mlp.experts.down_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.16.mlp.experts.gate_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.16.mlp.experts.up_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.17.mlp.experts.down_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.17.mlp.experts.gate_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.17.mlp.experts.up_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.18.mlp.experts.down_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.18.mlp.experts.gate_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.18.mlp.experts.up_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.19.mlp.experts.down_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.19.mlp.experts.gate_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.19.mlp.experts.up_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.2.mlp.experts.down_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.2.mlp.experts.gate_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.2.mlp.experts.up_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.20.mlp.experts.down_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.20.mlp.experts.gate_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.20.mlp.experts.up_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.21.mlp.experts.down_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.21.mlp.experts.gate_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.21.mlp.experts.up_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.22.mlp.experts.down_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.22.mlp.experts.gate_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.22.mlp.experts.up_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.23.mlp.experts.down_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.23.mlp.experts.gate_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.23.mlp.experts.up_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.3.mlp.experts.down_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.3.mlp.experts.gate_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.3.mlp.experts.up_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.4.mlp.experts.down_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.4.mlp.experts.gate_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.4.mlp.experts.up_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.5.mlp.experts.down_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.5.mlp.experts.gate_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.5.mlp.experts.up_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.6.mlp.experts.down_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.6.mlp.experts.gate_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.6.mlp.experts.up_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.7.mlp.experts.down_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.7.mlp.experts.gate_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.7.mlp.experts.up_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.8.mlp.experts.down_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.8.mlp.experts.gate_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.8.mlp.experts.up_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.9.mlp.experts.down_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.9.mlp.experts.gate_proj": {
      "bits": 4,
      "groupSize": 64
    },
    "model.model.layers.9.mlp.experts.up_proj": {
      "bits": 4,
      "groupSize": 64
    }
  },
  "tensors": {
    "model.model.layers.1.mlp.experts.down_proj.weight": {
      "offset": 0,
      "bytes": 50331648,
      "shape": [
        128,
        1536,
        64
      ],
      "dtype": "uint32"
    },
    "model.model.layers.1.mlp.experts.down_proj.scales": {
      "offset": 50331648,
      "bytes": 6291456,
      "shape": [
        128,
        1536,
        8
      ],
      "dtype": "float32"
    },
    "model.model.layers.1.mlp.experts.down_proj.biases": {
      "offset": 56623104,
      "bytes": 6291456,
      "shape": [
        128,
        1536,
        8
      ],
      "dtype": "float32"
    },
    "model.model.layers.1.mlp.experts.gate_proj.weight": {
      "offset": 62914560,
      "bytes": 50331648,
      "shape": [
        128,
        512,
        192
      ],
      "dtype": "uint32"
    },
    "model.model.layers.1.mlp.experts.gate_proj.scales": {
      "offset": 113246208,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.1.mlp.experts.gate_proj.biases": {
      "offset": 119537664,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.1.mlp.experts.up_proj.weight": {
      "offset": 125829120,
      "bytes": 50331648,
      "shape": [
        128,
        512,
        192
      ],
      "dtype": "uint32"
    },
    "model.model.layers.1.mlp.experts.up_proj.scales": {
      "offset": 176160768,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.1.mlp.experts.up_proj.biases": {
      "offset": 182452224,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.10.mlp.experts.down_proj.weight": {
      "offset": 188743680,
      "bytes": 50331648,
      "shape": [
        128,
        1536,
        64
      ],
      "dtype": "uint32"
    },
    "model.model.layers.10.mlp.experts.down_proj.scales": {
      "offset": 239075328,
      "bytes": 6291456,
      "shape": [
        128,
        1536,
        8
      ],
      "dtype": "float32"
    },
    "model.model.layers.10.mlp.experts.down_proj.biases": {
      "offset": 245366784,
      "bytes": 6291456,
      "shape": [
        128,
        1536,
        8
      ],
      "dtype": "float32"
    },
    "model.model.layers.10.mlp.experts.gate_proj.weight": {
      "offset": 251658240,
      "bytes": 50331648,
      "shape": [
        128,
        512,
        192
      ],
      "dtype": "uint32"
    },
    "model.model.layers.10.mlp.experts.gate_proj.scales": {
      "offset": 301989888,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.10.mlp.experts.gate_proj.biases": {
      "offset": 308281344,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.10.mlp.experts.up_proj.weight": {
      "offset": 314572800,
      "bytes": 50331648,
      "shape": [
        128,
        512,
        192
      ],
      "dtype": "uint32"
    },
    "model.model.layers.10.mlp.experts.up_proj.scales": {
      "offset": 364904448,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.10.mlp.experts.up_proj.biases": {
      "offset": 371195904,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.11.mlp.experts.down_proj.weight": {
      "offset": 377487360,
      "bytes": 50331648,
      "shape": [
        128,
        1536,
        64
      ],
      "dtype": "uint32"
    },
    "model.model.layers.11.mlp.experts.down_proj.scales": {
      "offset": 427819008,
      "bytes": 6291456,
      "shape": [
        128,
        1536,
        8
      ],
      "dtype": "float32"
    },
    "model.model.layers.11.mlp.experts.down_proj.biases": {
      "offset": 434110464,
      "bytes": 6291456,
      "shape": [
        128,
        1536,
        8
      ],
      "dtype": "float32"
    },
    "model.model.layers.11.mlp.experts.gate_proj.weight": {
      "offset": 440401920,
      "bytes": 50331648,
      "shape": [
        128,
        512,
        192
      ],
      "dtype": "uint32"
    },
    "model.model.layers.11.mlp.experts.gate_proj.scales": {
      "offset": 490733568,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.11.mlp.experts.gate_proj.biases": {
      "offset": 497025024,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.11.mlp.experts.up_proj.weight": {
      "offset": 503316480,
      "bytes": 50331648,
      "shape": [
        128,
        512,
        192
      ],
      "dtype": "uint32"
    },
    "model.model.layers.11.mlp.experts.up_proj.scales": {
      "offset": 553648128,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.11.mlp.experts.up_proj.biases": {
      "offset": 559939584,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.12.mlp.experts.down_proj.weight": {
      "offset": 566231040,
      "bytes": 50331648,
      "shape": [
        128,
        1536,
        64
      ],
      "dtype": "uint32"
    },
    "model.model.layers.12.mlp.experts.down_proj.scales": {
      "offset": 616562688,
      "bytes": 6291456,
      "shape": [
        128,
        1536,
        8
      ],
      "dtype": "float32"
    },
    "model.model.layers.12.mlp.experts.down_proj.biases": {
      "offset": 622854144,
      "bytes": 6291456,
      "shape": [
        128,
        1536,
        8
      ],
      "dtype": "float32"
    },
    "model.model.layers.12.mlp.experts.gate_proj.weight": {
      "offset": 629145600,
      "bytes": 50331648,
      "shape": [
        128,
        512,
        192
      ],
      "dtype": "uint32"
    },
    "model.model.layers.12.mlp.experts.gate_proj.scales": {
      "offset": 679477248,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.12.mlp.experts.gate_proj.biases": {
      "offset": 685768704,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.12.mlp.experts.up_proj.weight": {
      "offset": 692060160,
      "bytes": 50331648,
      "shape": [
        128,
        512,
        192
      ],
      "dtype": "uint32"
    },
    "model.model.layers.12.mlp.experts.up_proj.scales": {
      "offset": 742391808,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.12.mlp.experts.up_proj.biases": {
      "offset": 748683264,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.13.mlp.experts.down_proj.weight": {
      "offset": 754974720,
      "bytes": 50331648,
      "shape": [
        128,
        1536,
        64
      ],
      "dtype": "uint32"
    },
    "model.model.layers.13.mlp.experts.down_proj.scales": {
      "offset": 805306368,
      "bytes": 6291456,
      "shape": [
        128,
        1536,
        8
      ],
      "dtype": "float32"
    },
    "model.model.layers.13.mlp.experts.down_proj.biases": {
      "offset": 811597824,
      "bytes": 6291456,
      "shape": [
        128,
        1536,
        8
      ],
      "dtype": "float32"
    },
    "model.model.layers.13.mlp.experts.gate_proj.weight": {
      "offset": 817889280,
      "bytes": 50331648,
      "shape": [
        128,
        512,
        192
      ],
      "dtype": "uint32"
    },
    "model.model.layers.13.mlp.experts.gate_proj.scales": {
      "offset": 868220928,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.13.mlp.experts.gate_proj.biases": {
      "offset": 874512384,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.13.mlp.experts.up_proj.weight": {
      "offset": 880803840,
      "bytes": 50331648,
      "shape": [
        128,
        512,
        192
      ],
      "dtype": "uint32"
    },
    "model.model.layers.13.mlp.experts.up_proj.scales": {
      "offset": 931135488,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.13.mlp.experts.up_proj.biases": {
      "offset": 937426944,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.14.mlp.experts.down_proj.weight": {
      "offset": 943718400,
      "bytes": 50331648,
      "shape": [
        128,
        1536,
        64
      ],
      "dtype": "uint32"
    },
    "model.model.layers.14.mlp.experts.down_proj.scales": {
      "offset": 994050048,
      "bytes": 6291456,
      "shape": [
        128,
        1536,
        8
      ],
      "dtype": "float32"
    },
    "model.model.layers.14.mlp.experts.down_proj.biases": {
      "offset": 1000341504,
      "bytes": 6291456,
      "shape": [
        128,
        1536,
        8
      ],
      "dtype": "float32"
    },
    "model.model.layers.14.mlp.experts.gate_proj.weight": {
      "offset": 1006632960,
      "bytes": 50331648,
      "shape": [
        128,
        512,
        192
      ],
      "dtype": "uint32"
    },
    "model.model.layers.14.mlp.experts.gate_proj.scales": {
      "offset": 1056964608,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.14.mlp.experts.gate_proj.biases": {
      "offset": 1063256064,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.14.mlp.experts.up_proj.weight": {
      "offset": 1069547520,
      "bytes": 50331648,
      "shape": [
        128,
        512,
        192
      ],
      "dtype": "uint32"
    },
    "model.model.layers.14.mlp.experts.up_proj.scales": {
      "offset": 1119879168,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.14.mlp.experts.up_proj.biases": {
      "offset": 1126170624,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.15.mlp.experts.down_proj.weight": {
      "offset": 1132462080,
      "bytes": 50331648,
      "shape": [
        128,
        1536,
        64
      ],
      "dtype": "uint32"
    },
    "model.model.layers.15.mlp.experts.down_proj.scales": {
      "offset": 1182793728,
      "bytes": 6291456,
      "shape": [
        128,
        1536,
        8
      ],
      "dtype": "float32"
    },
    "model.model.layers.15.mlp.experts.down_proj.biases": {
      "offset": 1189085184,
      "bytes": 6291456,
      "shape": [
        128,
        1536,
        8
      ],
      "dtype": "float32"
    },
    "model.model.layers.15.mlp.experts.gate_proj.weight": {
      "offset": 1195376640,
      "bytes": 50331648,
      "shape": [
        128,
        512,
        192
      ],
      "dtype": "uint32"
    },
    "model.model.layers.15.mlp.experts.gate_proj.scales": {
      "offset": 1245708288,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.15.mlp.experts.gate_proj.biases": {
      "offset": 1251999744,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.15.mlp.experts.up_proj.weight": {
      "offset": 1258291200,
      "bytes": 50331648,
      "shape": [
        128,
        512,
        192
      ],
      "dtype": "uint32"
    },
    "model.model.layers.15.mlp.experts.up_proj.scales": {
      "offset": 1308622848,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.15.mlp.experts.up_proj.biases": {
      "offset": 1314914304,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.16.mlp.experts.down_proj.weight": {
      "offset": 1321205760,
      "bytes": 50331648,
      "shape": [
        128,
        1536,
        64
      ],
      "dtype": "uint32"
    },
    "model.model.layers.16.mlp.experts.down_proj.scales": {
      "offset": 1371537408,
      "bytes": 6291456,
      "shape": [
        128,
        1536,
        8
      ],
      "dtype": "float32"
    },
    "model.model.layers.16.mlp.experts.down_proj.biases": {
      "offset": 1377828864,
      "bytes": 6291456,
      "shape": [
        128,
        1536,
        8
      ],
      "dtype": "float32"
    },
    "model.model.layers.16.mlp.experts.gate_proj.weight": {
      "offset": 1384120320,
      "bytes": 50331648,
      "shape": [
        128,
        512,
        192
      ],
      "dtype": "uint32"
    },
    "model.model.layers.16.mlp.experts.gate_proj.scales": {
      "offset": 1434451968,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.16.mlp.experts.gate_proj.biases": {
      "offset": 1440743424,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.16.mlp.experts.up_proj.weight": {
      "offset": 1447034880,
      "bytes": 50331648,
      "shape": [
        128,
        512,
        192
      ],
      "dtype": "uint32"
    },
    "model.model.layers.16.mlp.experts.up_proj.scales": {
      "offset": 1497366528,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.16.mlp.experts.up_proj.biases": {
      "offset": 1503657984,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.17.mlp.experts.down_proj.weight": {
      "offset": 1509949440,
      "bytes": 50331648,
      "shape": [
        128,
        1536,
        64
      ],
      "dtype": "uint32"
    },
    "model.model.layers.17.mlp.experts.down_proj.scales": {
      "offset": 1560281088,
      "bytes": 6291456,
      "shape": [
        128,
        1536,
        8
      ],
      "dtype": "float32"
    },
    "model.model.layers.17.mlp.experts.down_proj.biases": {
      "offset": 1566572544,
      "bytes": 6291456,
      "shape": [
        128,
        1536,
        8
      ],
      "dtype": "float32"
    },
    "model.model.layers.17.mlp.experts.gate_proj.weight": {
      "offset": 1572864000,
      "bytes": 50331648,
      "shape": [
        128,
        512,
        192
      ],
      "dtype": "uint32"
    },
    "model.model.layers.17.mlp.experts.gate_proj.scales": {
      "offset": 1623195648,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.17.mlp.experts.gate_proj.biases": {
      "offset": 1629487104,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.17.mlp.experts.up_proj.weight": {
      "offset": 1635778560,
      "bytes": 50331648,
      "shape": [
        128,
        512,
        192
      ],
      "dtype": "uint32"
    },
    "model.model.layers.17.mlp.experts.up_proj.scales": {
      "offset": 1686110208,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.17.mlp.experts.up_proj.biases": {
      "offset": 1692401664,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.18.mlp.experts.down_proj.weight": {
      "offset": 1698693120,
      "bytes": 50331648,
      "shape": [
        128,
        1536,
        64
      ],
      "dtype": "uint32"
    },
    "model.model.layers.18.mlp.experts.down_proj.scales": {
      "offset": 1749024768,
      "bytes": 6291456,
      "shape": [
        128,
        1536,
        8
      ],
      "dtype": "float32"
    },
    "model.model.layers.18.mlp.experts.down_proj.biases": {
      "offset": 1755316224,
      "bytes": 6291456,
      "shape": [
        128,
        1536,
        8
      ],
      "dtype": "float32"
    },
    "model.model.layers.18.mlp.experts.gate_proj.weight": {
      "offset": 1761607680,
      "bytes": 50331648,
      "shape": [
        128,
        512,
        192
      ],
      "dtype": "uint32"
    },
    "model.model.layers.18.mlp.experts.gate_proj.scales": {
      "offset": 1811939328,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.18.mlp.experts.gate_proj.biases": {
      "offset": 1818230784,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.18.mlp.experts.up_proj.weight": {
      "offset": 1824522240,
      "bytes": 50331648,
      "shape": [
        128,
        512,
        192
      ],
      "dtype": "uint32"
    },
    "model.model.layers.18.mlp.experts.up_proj.scales": {
      "offset": 1874853888,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.18.mlp.experts.up_proj.biases": {
      "offset": 1881145344,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.19.mlp.experts.down_proj.weight": {
      "offset": 1887436800,
      "bytes": 50331648,
      "shape": [
        128,
        1536,
        64
      ],
      "dtype": "uint32"
    },
    "model.model.layers.19.mlp.experts.down_proj.scales": {
      "offset": 1937768448,
      "bytes": 6291456,
      "shape": [
        128,
        1536,
        8
      ],
      "dtype": "float32"
    },
    "model.model.layers.19.mlp.experts.down_proj.biases": {
      "offset": 1944059904,
      "bytes": 6291456,
      "shape": [
        128,
        1536,
        8
      ],
      "dtype": "float32"
    },
    "model.model.layers.19.mlp.experts.gate_proj.weight": {
      "offset": 1950351360,
      "bytes": 50331648,
      "shape": [
        128,
        512,
        192
      ],
      "dtype": "uint32"
    },
    "model.model.layers.19.mlp.experts.gate_proj.scales": {
      "offset": 2000683008,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.19.mlp.experts.gate_proj.biases": {
      "offset": 2006974464,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.19.mlp.experts.up_proj.weight": {
      "offset": 2013265920,
      "bytes": 50331648,
      "shape": [
        128,
        512,
        192
      ],
      "dtype": "uint32"
    },
    "model.model.layers.19.mlp.experts.up_proj.scales": {
      "offset": 2063597568,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.19.mlp.experts.up_proj.biases": {
      "offset": 2069889024,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.2.mlp.experts.down_proj.weight": {
      "offset": 2076180480,
      "bytes": 50331648,
      "shape": [
        128,
        1536,
        64
      ],
      "dtype": "uint32"
    },
    "model.model.layers.2.mlp.experts.down_proj.scales": {
      "offset": 2126512128,
      "bytes": 6291456,
      "shape": [
        128,
        1536,
        8
      ],
      "dtype": "float32"
    },
    "model.model.layers.2.mlp.experts.down_proj.biases": {
      "offset": 2132803584,
      "bytes": 6291456,
      "shape": [
        128,
        1536,
        8
      ],
      "dtype": "float32"
    },
    "model.model.layers.2.mlp.experts.gate_proj.weight": {
      "offset": 2139095040,
      "bytes": 50331648,
      "shape": [
        128,
        512,
        192
      ],
      "dtype": "uint32"
    },
    "model.model.layers.2.mlp.experts.gate_proj.scales": {
      "offset": 2189426688,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.2.mlp.experts.gate_proj.biases": {
      "offset": 2195718144,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.2.mlp.experts.up_proj.weight": {
      "offset": 2202009600,
      "bytes": 50331648,
      "shape": [
        128,
        512,
        192
      ],
      "dtype": "uint32"
    },
    "model.model.layers.2.mlp.experts.up_proj.scales": {
      "offset": 2252341248,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.2.mlp.experts.up_proj.biases": {
      "offset": 2258632704,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.20.mlp.experts.down_proj.weight": {
      "offset": 2264924160,
      "bytes": 50331648,
      "shape": [
        128,
        1536,
        64
      ],
      "dtype": "uint32"
    },
    "model.model.layers.20.mlp.experts.down_proj.scales": {
      "offset": 2315255808,
      "bytes": 6291456,
      "shape": [
        128,
        1536,
        8
      ],
      "dtype": "float32"
    },
    "model.model.layers.20.mlp.experts.down_proj.biases": {
      "offset": 2321547264,
      "bytes": 6291456,
      "shape": [
        128,
        1536,
        8
      ],
      "dtype": "float32"
    },
    "model.model.layers.20.mlp.experts.gate_proj.weight": {
      "offset": 2327838720,
      "bytes": 50331648,
      "shape": [
        128,
        512,
        192
      ],
      "dtype": "uint32"
    },
    "model.model.layers.20.mlp.experts.gate_proj.scales": {
      "offset": 2378170368,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.20.mlp.experts.gate_proj.biases": {
      "offset": 2384461824,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.20.mlp.experts.up_proj.weight": {
      "offset": 2390753280,
      "bytes": 50331648,
      "shape": [
        128,
        512,
        192
      ],
      "dtype": "uint32"
    },
    "model.model.layers.20.mlp.experts.up_proj.scales": {
      "offset": 2441084928,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.20.mlp.experts.up_proj.biases": {
      "offset": 2447376384,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.21.mlp.experts.down_proj.weight": {
      "offset": 2453667840,
      "bytes": 50331648,
      "shape": [
        128,
        1536,
        64
      ],
      "dtype": "uint32"
    },
    "model.model.layers.21.mlp.experts.down_proj.scales": {
      "offset": 2503999488,
      "bytes": 6291456,
      "shape": [
        128,
        1536,
        8
      ],
      "dtype": "float32"
    },
    "model.model.layers.21.mlp.experts.down_proj.biases": {
      "offset": 2510290944,
      "bytes": 6291456,
      "shape": [
        128,
        1536,
        8
      ],
      "dtype": "float32"
    },
    "model.model.layers.21.mlp.experts.gate_proj.weight": {
      "offset": 2516582400,
      "bytes": 50331648,
      "shape": [
        128,
        512,
        192
      ],
      "dtype": "uint32"
    },
    "model.model.layers.21.mlp.experts.gate_proj.scales": {
      "offset": 2566914048,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.21.mlp.experts.gate_proj.biases": {
      "offset": 2573205504,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.21.mlp.experts.up_proj.weight": {
      "offset": 2579496960,
      "bytes": 50331648,
      "shape": [
        128,
        512,
        192
      ],
      "dtype": "uint32"
    },
    "model.model.layers.21.mlp.experts.up_proj.scales": {
      "offset": 2629828608,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.21.mlp.experts.up_proj.biases": {
      "offset": 2636120064,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.22.mlp.experts.down_proj.weight": {
      "offset": 2642411520,
      "bytes": 50331648,
      "shape": [
        128,
        1536,
        64
      ],
      "dtype": "uint32"
    },
    "model.model.layers.22.mlp.experts.down_proj.scales": {
      "offset": 2692743168,
      "bytes": 6291456,
      "shape": [
        128,
        1536,
        8
      ],
      "dtype": "float32"
    },
    "model.model.layers.22.mlp.experts.down_proj.biases": {
      "offset": 2699034624,
      "bytes": 6291456,
      "shape": [
        128,
        1536,
        8
      ],
      "dtype": "float32"
    },
    "model.model.layers.22.mlp.experts.gate_proj.weight": {
      "offset": 2705326080,
      "bytes": 50331648,
      "shape": [
        128,
        512,
        192
      ],
      "dtype": "uint32"
    },
    "model.model.layers.22.mlp.experts.gate_proj.scales": {
      "offset": 2755657728,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.22.mlp.experts.gate_proj.biases": {
      "offset": 2761949184,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.22.mlp.experts.up_proj.weight": {
      "offset": 2768240640,
      "bytes": 50331648,
      "shape": [
        128,
        512,
        192
      ],
      "dtype": "uint32"
    },
    "model.model.layers.22.mlp.experts.up_proj.scales": {
      "offset": 2818572288,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.22.mlp.experts.up_proj.biases": {
      "offset": 2824863744,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.23.mlp.experts.down_proj.weight": {
      "offset": 2831155200,
      "bytes": 50331648,
      "shape": [
        128,
        1536,
        64
      ],
      "dtype": "uint32"
    },
    "model.model.layers.23.mlp.experts.down_proj.scales": {
      "offset": 2881486848,
      "bytes": 6291456,
      "shape": [
        128,
        1536,
        8
      ],
      "dtype": "float32"
    },
    "model.model.layers.23.mlp.experts.down_proj.biases": {
      "offset": 2887778304,
      "bytes": 6291456,
      "shape": [
        128,
        1536,
        8
      ],
      "dtype": "float32"
    },
    "model.model.layers.23.mlp.experts.gate_proj.weight": {
      "offset": 2894069760,
      "bytes": 50331648,
      "shape": [
        128,
        512,
        192
      ],
      "dtype": "uint32"
    },
    "model.model.layers.23.mlp.experts.gate_proj.scales": {
      "offset": 2944401408,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.23.mlp.experts.gate_proj.biases": {
      "offset": 2950692864,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.23.mlp.experts.up_proj.weight": {
      "offset": 2956984320,
      "bytes": 50331648,
      "shape": [
        128,
        512,
        192
      ],
      "dtype": "uint32"
    },
    "model.model.layers.23.mlp.experts.up_proj.scales": {
      "offset": 3007315968,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.23.mlp.experts.up_proj.biases": {
      "offset": 3013607424,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.3.mlp.experts.down_proj.weight": {
      "offset": 3019898880,
      "bytes": 50331648,
      "shape": [
        128,
        1536,
        64
      ],
      "dtype": "uint32"
    },
    "model.model.layers.3.mlp.experts.down_proj.scales": {
      "offset": 3070230528,
      "bytes": 6291456,
      "shape": [
        128,
        1536,
        8
      ],
      "dtype": "float32"
    },
    "model.model.layers.3.mlp.experts.down_proj.biases": {
      "offset": 3076521984,
      "bytes": 6291456,
      "shape": [
        128,
        1536,
        8
      ],
      "dtype": "float32"
    },
    "model.model.layers.3.mlp.experts.gate_proj.weight": {
      "offset": 3082813440,
      "bytes": 50331648,
      "shape": [
        128,
        512,
        192
      ],
      "dtype": "uint32"
    },
    "model.model.layers.3.mlp.experts.gate_proj.scales": {
      "offset": 3133145088,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.3.mlp.experts.gate_proj.biases": {
      "offset": 3139436544,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.3.mlp.experts.up_proj.weight": {
      "offset": 3145728000,
      "bytes": 50331648,
      "shape": [
        128,
        512,
        192
      ],
      "dtype": "uint32"
    },
    "model.model.layers.3.mlp.experts.up_proj.scales": {
      "offset": 3196059648,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.3.mlp.experts.up_proj.biases": {
      "offset": 3202351104,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.4.mlp.experts.down_proj.weight": {
      "offset": 3208642560,
      "bytes": 50331648,
      "shape": [
        128,
        1536,
        64
      ],
      "dtype": "uint32"
    },
    "model.model.layers.4.mlp.experts.down_proj.scales": {
      "offset": 3258974208,
      "bytes": 6291456,
      "shape": [
        128,
        1536,
        8
      ],
      "dtype": "float32"
    },
    "model.model.layers.4.mlp.experts.down_proj.biases": {
      "offset": 3265265664,
      "bytes": 6291456,
      "shape": [
        128,
        1536,
        8
      ],
      "dtype": "float32"
    },
    "model.model.layers.4.mlp.experts.gate_proj.weight": {
      "offset": 3271557120,
      "bytes": 50331648,
      "shape": [
        128,
        512,
        192
      ],
      "dtype": "uint32"
    },
    "model.model.layers.4.mlp.experts.gate_proj.scales": {
      "offset": 3321888768,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.4.mlp.experts.gate_proj.biases": {
      "offset": 3328180224,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.4.mlp.experts.up_proj.weight": {
      "offset": 3334471680,
      "bytes": 50331648,
      "shape": [
        128,
        512,
        192
      ],
      "dtype": "uint32"
    },
    "model.model.layers.4.mlp.experts.up_proj.scales": {
      "offset": 3384803328,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.4.mlp.experts.up_proj.biases": {
      "offset": 3391094784,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.5.mlp.experts.down_proj.weight": {
      "offset": 3397386240,
      "bytes": 50331648,
      "shape": [
        128,
        1536,
        64
      ],
      "dtype": "uint32"
    },
    "model.model.layers.5.mlp.experts.down_proj.scales": {
      "offset": 3447717888,
      "bytes": 6291456,
      "shape": [
        128,
        1536,
        8
      ],
      "dtype": "float32"
    },
    "model.model.layers.5.mlp.experts.down_proj.biases": {
      "offset": 3454009344,
      "bytes": 6291456,
      "shape": [
        128,
        1536,
        8
      ],
      "dtype": "float32"
    },
    "model.model.layers.5.mlp.experts.gate_proj.weight": {
      "offset": 3460300800,
      "bytes": 50331648,
      "shape": [
        128,
        512,
        192
      ],
      "dtype": "uint32"
    },
    "model.model.layers.5.mlp.experts.gate_proj.scales": {
      "offset": 3510632448,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.5.mlp.experts.gate_proj.biases": {
      "offset": 3516923904,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.5.mlp.experts.up_proj.weight": {
      "offset": 3523215360,
      "bytes": 50331648,
      "shape": [
        128,
        512,
        192
      ],
      "dtype": "uint32"
    },
    "model.model.layers.5.mlp.experts.up_proj.scales": {
      "offset": 3573547008,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.5.mlp.experts.up_proj.biases": {
      "offset": 3579838464,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.6.mlp.experts.down_proj.weight": {
      "offset": 3586129920,
      "bytes": 50331648,
      "shape": [
        128,
        1536,
        64
      ],
      "dtype": "uint32"
    },
    "model.model.layers.6.mlp.experts.down_proj.scales": {
      "offset": 3636461568,
      "bytes": 6291456,
      "shape": [
        128,
        1536,
        8
      ],
      "dtype": "float32"
    },
    "model.model.layers.6.mlp.experts.down_proj.biases": {
      "offset": 3642753024,
      "bytes": 6291456,
      "shape": [
        128,
        1536,
        8
      ],
      "dtype": "float32"
    },
    "model.model.layers.6.mlp.experts.gate_proj.weight": {
      "offset": 3649044480,
      "bytes": 50331648,
      "shape": [
        128,
        512,
        192
      ],
      "dtype": "uint32"
    },
    "model.model.layers.6.mlp.experts.gate_proj.scales": {
      "offset": 3699376128,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.6.mlp.experts.gate_proj.biases": {
      "offset": 3705667584,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.6.mlp.experts.up_proj.weight": {
      "offset": 3711959040,
      "bytes": 50331648,
      "shape": [
        128,
        512,
        192
      ],
      "dtype": "uint32"
    },
    "model.model.layers.6.mlp.experts.up_proj.scales": {
      "offset": 3762290688,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.6.mlp.experts.up_proj.biases": {
      "offset": 3768582144,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.7.mlp.experts.down_proj.weight": {
      "offset": 3774873600,
      "bytes": 50331648,
      "shape": [
        128,
        1536,
        64
      ],
      "dtype": "uint32"
    },
    "model.model.layers.7.mlp.experts.down_proj.scales": {
      "offset": 3825205248,
      "bytes": 6291456,
      "shape": [
        128,
        1536,
        8
      ],
      "dtype": "float32"
    },
    "model.model.layers.7.mlp.experts.down_proj.biases": {
      "offset": 3831496704,
      "bytes": 6291456,
      "shape": [
        128,
        1536,
        8
      ],
      "dtype": "float32"
    },
    "model.model.layers.7.mlp.experts.gate_proj.weight": {
      "offset": 3837788160,
      "bytes": 50331648,
      "shape": [
        128,
        512,
        192
      ],
      "dtype": "uint32"
    },
    "model.model.layers.7.mlp.experts.gate_proj.scales": {
      "offset": 3888119808,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.7.mlp.experts.gate_proj.biases": {
      "offset": 3894411264,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.7.mlp.experts.up_proj.weight": {
      "offset": 3900702720,
      "bytes": 50331648,
      "shape": [
        128,
        512,
        192
      ],
      "dtype": "uint32"
    },
    "model.model.layers.7.mlp.experts.up_proj.scales": {
      "offset": 3951034368,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.7.mlp.experts.up_proj.biases": {
      "offset": 3957325824,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.8.mlp.experts.down_proj.weight": {
      "offset": 3963617280,
      "bytes": 50331648,
      "shape": [
        128,
        1536,
        64
      ],
      "dtype": "uint32"
    },
    "model.model.layers.8.mlp.experts.down_proj.scales": {
      "offset": 4013948928,
      "bytes": 6291456,
      "shape": [
        128,
        1536,
        8
      ],
      "dtype": "float32"
    },
    "model.model.layers.8.mlp.experts.down_proj.biases": {
      "offset": 4020240384,
      "bytes": 6291456,
      "shape": [
        128,
        1536,
        8
      ],
      "dtype": "float32"
    },
    "model.model.layers.8.mlp.experts.gate_proj.weight": {
      "offset": 4026531840,
      "bytes": 50331648,
      "shape": [
        128,
        512,
        192
      ],
      "dtype": "uint32"
    },
    "model.model.layers.8.mlp.experts.gate_proj.scales": {
      "offset": 4076863488,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.8.mlp.experts.gate_proj.biases": {
      "offset": 4083154944,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.8.mlp.experts.up_proj.weight": {
      "offset": 4089446400,
      "bytes": 50331648,
      "shape": [
        128,
        512,
        192
      ],
      "dtype": "uint32"
    },
    "model.model.layers.8.mlp.experts.up_proj.scales": {
      "offset": 4139778048,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.8.mlp.experts.up_proj.biases": {
      "offset": 4146069504,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.9.mlp.experts.down_proj.weight": {
      "offset": 4152360960,
      "bytes": 50331648,
      "shape": [
        128,
        1536,
        64
      ],
      "dtype": "uint32"
    },
    "model.model.layers.9.mlp.experts.down_proj.scales": {
      "offset": 4202692608,
      "bytes": 6291456,
      "shape": [
        128,
        1536,
        8
      ],
      "dtype": "float32"
    },
    "model.model.layers.9.mlp.experts.down_proj.biases": {
      "offset": 4208984064,
      "bytes": 6291456,
      "shape": [
        128,
        1536,
        8
      ],
      "dtype": "float32"
    },
    "model.model.layers.9.mlp.experts.gate_proj.weight": {
      "offset": 4215275520,
      "bytes": 50331648,
      "shape": [
        128,
        512,
        192
      ],
      "dtype": "uint32"
    },
    "model.model.layers.9.mlp.experts.gate_proj.scales": {
      "offset": 4265607168,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.9.mlp.experts.gate_proj.biases": {
      "offset": 4271898624,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.9.mlp.experts.up_proj.weight": {
      "offset": 4278190080,
      "bytes": 50331648,
      "shape": [
        128,
        512,
        192
      ],
      "dtype": "uint32"
    },
    "model.model.layers.9.mlp.experts.up_proj.scales": {
      "offset": 4328521728,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.9.mlp.experts.up_proj.biases": {
      "offset": 4334813184,
      "bytes": 6291456,
      "shape": [
        128,
        512,
        24
      ],
      "dtype": "float32"
    },
    "model.model.layers.0.attention.A_log": {
      "offset": 4341104640,
      "bytes": 64,
      "shape": [
        16
      ],
      "dtype": "float32"
    },
    "model.model.layers.0.attention.b_proj.weight": {
      "offset": 4341121024,
      "bytes": 49152,
      "shape": [
        16,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.0.attention.dt_bias": {
      "offset": 4341170176,
      "bytes": 8192,
      "shape": [
        2048
      ],
      "dtype": "float32"
    },
    "model.model.layers.0.attention.f_proj.weight": {
      "offset": 4341186560,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.0.attention.g_proj.weight": {
      "offset": 4347478016,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.0.attention.k_conv1d.weight": {
      "offset": 4353769472,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.0.attention.k_proj.weight": {
      "offset": 4353785856,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.0.attention.o_norm.weight": {
      "offset": 4360077312,
      "bytes": 256,
      "shape": [
        128
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.0.attention.o_proj.weight": {
      "offset": 4360093696,
      "bytes": 6291456,
      "shape": [
        1536,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.0.attention.q_conv1d.weight": {
      "offset": 4366385152,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.0.attention.q_proj.weight": {
      "offset": 4366401536,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.0.attention.v_conv1d.weight": {
      "offset": 4372692992,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.0.attention.v_proj.weight": {
      "offset": 4372709376,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.0.input_layernorm.weight": {
      "offset": 4379000832,
      "bytes": 3072,
      "shape": [
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.0.mlp.down_proj.weight": {
      "offset": 4379017216,
      "bytes": 14155776,
      "shape": [
        1536,
        4608
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.0.mlp.gate_proj.weight": {
      "offset": 4393172992,
      "bytes": 14155776,
      "shape": [
        4608,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.0.mlp.up_proj.weight": {
      "offset": 4407328768,
      "bytes": 14155776,
      "shape": [
        4608,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.0.post_attention_layernorm.weight": {
      "offset": 4421484544,
      "bytes": 3072,
      "shape": [
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.1.attention.A_log": {
      "offset": 4421500928,
      "bytes": 64,
      "shape": [
        16
      ],
      "dtype": "float32"
    },
    "model.model.layers.1.attention.b_proj.weight": {
      "offset": 4421517312,
      "bytes": 49152,
      "shape": [
        16,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.1.attention.dt_bias": {
      "offset": 4421566464,
      "bytes": 8192,
      "shape": [
        2048
      ],
      "dtype": "float32"
    },
    "model.model.layers.1.attention.f_proj.weight": {
      "offset": 4421582848,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.1.attention.g_proj.weight": {
      "offset": 4427874304,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.1.attention.k_conv1d.weight": {
      "offset": 4434165760,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.1.attention.k_proj.weight": {
      "offset": 4434182144,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.1.attention.o_norm.weight": {
      "offset": 4440473600,
      "bytes": 256,
      "shape": [
        128
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.1.attention.o_proj.weight": {
      "offset": 4440489984,
      "bytes": 6291456,
      "shape": [
        1536,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.1.attention.q_conv1d.weight": {
      "offset": 4446781440,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.1.attention.q_proj.weight": {
      "offset": 4446797824,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.1.attention.v_conv1d.weight": {
      "offset": 4453089280,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.1.attention.v_proj.weight": {
      "offset": 4453105664,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.1.input_layernorm.weight": {
      "offset": 4459397120,
      "bytes": 3072,
      "shape": [
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.1.mlp.gate.expert_bias": {
      "offset": 4459413504,
      "bytes": 512,
      "shape": [
        128
      ],
      "dtype": "float32"
    },
    "model.model.layers.1.mlp.gate.weight": {
      "offset": 4459429888,
      "bytes": 786432,
      "shape": [
        128,
        1536
      ],
      "dtype": "float32"
    },
    "model.model.layers.1.mlp.shared_experts.down_proj.weight": {
      "offset": 4460216320,
      "bytes": 1572864,
      "shape": [
        1536,
        512
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.1.mlp.shared_experts.gate_proj.weight": {
      "offset": 4461789184,
      "bytes": 1572864,
      "shape": [
        512,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.1.mlp.shared_experts.up_proj.weight": {
      "offset": 4463362048,
      "bytes": 1572864,
      "shape": [
        512,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.1.post_attention_layernorm.weight": {
      "offset": 4464934912,
      "bytes": 3072,
      "shape": [
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.10.attention.A_log": {
      "offset": 4464951296,
      "bytes": 64,
      "shape": [
        16
      ],
      "dtype": "float32"
    },
    "model.model.layers.10.attention.b_proj.weight": {
      "offset": 4464967680,
      "bytes": 49152,
      "shape": [
        16,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.10.attention.dt_bias": {
      "offset": 4465016832,
      "bytes": 8192,
      "shape": [
        2048
      ],
      "dtype": "float32"
    },
    "model.model.layers.10.attention.f_proj.weight": {
      "offset": 4465033216,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.10.attention.g_proj.weight": {
      "offset": 4471324672,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.10.attention.k_conv1d.weight": {
      "offset": 4477616128,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.10.attention.k_proj.weight": {
      "offset": 4477632512,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.10.attention.o_norm.weight": {
      "offset": 4483923968,
      "bytes": 256,
      "shape": [
        128
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.10.attention.o_proj.weight": {
      "offset": 4483940352,
      "bytes": 6291456,
      "shape": [
        1536,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.10.attention.q_conv1d.weight": {
      "offset": 4490231808,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.10.attention.q_proj.weight": {
      "offset": 4490248192,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.10.attention.v_conv1d.weight": {
      "offset": 4496539648,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.10.attention.v_proj.weight": {
      "offset": 4496556032,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.10.input_layernorm.weight": {
      "offset": 4502847488,
      "bytes": 3072,
      "shape": [
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.10.mlp.gate.expert_bias": {
      "offset": 4502863872,
      "bytes": 512,
      "shape": [
        128
      ],
      "dtype": "float32"
    },
    "model.model.layers.10.mlp.gate.weight": {
      "offset": 4502880256,
      "bytes": 786432,
      "shape": [
        128,
        1536
      ],
      "dtype": "float32"
    },
    "model.model.layers.10.mlp.shared_experts.down_proj.weight": {
      "offset": 4503666688,
      "bytes": 1572864,
      "shape": [
        1536,
        512
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.10.mlp.shared_experts.gate_proj.weight": {
      "offset": 4505239552,
      "bytes": 1572864,
      "shape": [
        512,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.10.mlp.shared_experts.up_proj.weight": {
      "offset": 4506812416,
      "bytes": 1572864,
      "shape": [
        512,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.10.post_attention_layernorm.weight": {
      "offset": 4508385280,
      "bytes": 3072,
      "shape": [
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.11.attention.dense.weight": {
      "offset": 4508401664,
      "bytes": 6291456,
      "shape": [
        1536,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.11.attention.g_proj.weight": {
      "offset": 4514693120,
      "bytes": 49152,
      "shape": [
        16,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.11.attention.kv_a_layernorm.weight": {
      "offset": 4514742272,
      "bytes": 1024,
      "shape": [
        512
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.11.attention.kv_a_proj_with_mqa.weight": {
      "offset": 4514758656,
      "bytes": 1769472,
      "shape": [
        576,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.11.attention.kv_b_proj.weight": {
      "offset": 4516528128,
      "bytes": 4194304,
      "shape": [
        4096,
        512
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.11.attention.q_a_layernorm.weight": {
      "offset": 4520722432,
      "bytes": 512,
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.11.attention.q_a_proj.weight": {
      "offset": 4520738816,
      "bytes": 786432,
      "shape": [
        256,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.11.attention.q_b_proj.weight": {
      "offset": 4521525248,
      "bytes": 1572864,
      "shape": [
        3072,
        256
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.11.input_layernorm.weight": {
      "offset": 4523098112,
      "bytes": 3072,
      "shape": [
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.11.mlp.gate.expert_bias": {
      "offset": 4523114496,
      "bytes": 512,
      "shape": [
        128
      ],
      "dtype": "float32"
    },
    "model.model.layers.11.mlp.gate.weight": {
      "offset": 4523130880,
      "bytes": 786432,
      "shape": [
        128,
        1536
      ],
      "dtype": "float32"
    },
    "model.model.layers.11.mlp.shared_experts.down_proj.weight": {
      "offset": 4523917312,
      "bytes": 1572864,
      "shape": [
        1536,
        512
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.11.mlp.shared_experts.gate_proj.weight": {
      "offset": 4525490176,
      "bytes": 1572864,
      "shape": [
        512,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.11.mlp.shared_experts.up_proj.weight": {
      "offset": 4527063040,
      "bytes": 1572864,
      "shape": [
        512,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.11.post_attention_layernorm.weight": {
      "offset": 4528635904,
      "bytes": 3072,
      "shape": [
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.12.attention.A_log": {
      "offset": 4528652288,
      "bytes": 64,
      "shape": [
        16
      ],
      "dtype": "float32"
    },
    "model.model.layers.12.attention.b_proj.weight": {
      "offset": 4528668672,
      "bytes": 49152,
      "shape": [
        16,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.12.attention.dt_bias": {
      "offset": 4528717824,
      "bytes": 8192,
      "shape": [
        2048
      ],
      "dtype": "float32"
    },
    "model.model.layers.12.attention.f_proj.weight": {
      "offset": 4528734208,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.12.attention.g_proj.weight": {
      "offset": 4535025664,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.12.attention.k_conv1d.weight": {
      "offset": 4541317120,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.12.attention.k_proj.weight": {
      "offset": 4541333504,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.12.attention.o_norm.weight": {
      "offset": 4547624960,
      "bytes": 256,
      "shape": [
        128
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.12.attention.o_proj.weight": {
      "offset": 4547641344,
      "bytes": 6291456,
      "shape": [
        1536,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.12.attention.q_conv1d.weight": {
      "offset": 4553932800,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.12.attention.q_proj.weight": {
      "offset": 4553949184,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.12.attention.v_conv1d.weight": {
      "offset": 4560240640,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.12.attention.v_proj.weight": {
      "offset": 4560257024,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.12.input_layernorm.weight": {
      "offset": 4566548480,
      "bytes": 3072,
      "shape": [
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.12.mlp.gate.expert_bias": {
      "offset": 4566564864,
      "bytes": 512,
      "shape": [
        128
      ],
      "dtype": "float32"
    },
    "model.model.layers.12.mlp.gate.weight": {
      "offset": 4566581248,
      "bytes": 786432,
      "shape": [
        128,
        1536
      ],
      "dtype": "float32"
    },
    "model.model.layers.12.mlp.shared_experts.down_proj.weight": {
      "offset": 4567367680,
      "bytes": 1572864,
      "shape": [
        1536,
        512
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.12.mlp.shared_experts.gate_proj.weight": {
      "offset": 4568940544,
      "bytes": 1572864,
      "shape": [
        512,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.12.mlp.shared_experts.up_proj.weight": {
      "offset": 4570513408,
      "bytes": 1572864,
      "shape": [
        512,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.12.post_attention_layernorm.weight": {
      "offset": 4572086272,
      "bytes": 3072,
      "shape": [
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.13.attention.A_log": {
      "offset": 4572102656,
      "bytes": 64,
      "shape": [
        16
      ],
      "dtype": "float32"
    },
    "model.model.layers.13.attention.b_proj.weight": {
      "offset": 4572119040,
      "bytes": 49152,
      "shape": [
        16,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.13.attention.dt_bias": {
      "offset": 4572168192,
      "bytes": 8192,
      "shape": [
        2048
      ],
      "dtype": "float32"
    },
    "model.model.layers.13.attention.f_proj.weight": {
      "offset": 4572184576,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.13.attention.g_proj.weight": {
      "offset": 4578476032,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.13.attention.k_conv1d.weight": {
      "offset": 4584767488,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.13.attention.k_proj.weight": {
      "offset": 4584783872,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.13.attention.o_norm.weight": {
      "offset": 4591075328,
      "bytes": 256,
      "shape": [
        128
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.13.attention.o_proj.weight": {
      "offset": 4591091712,
      "bytes": 6291456,
      "shape": [
        1536,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.13.attention.q_conv1d.weight": {
      "offset": 4597383168,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.13.attention.q_proj.weight": {
      "offset": 4597399552,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.13.attention.v_conv1d.weight": {
      "offset": 4603691008,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.13.attention.v_proj.weight": {
      "offset": 4603707392,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.13.input_layernorm.weight": {
      "offset": 4609998848,
      "bytes": 3072,
      "shape": [
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.13.mlp.gate.expert_bias": {
      "offset": 4610015232,
      "bytes": 512,
      "shape": [
        128
      ],
      "dtype": "float32"
    },
    "model.model.layers.13.mlp.gate.weight": {
      "offset": 4610031616,
      "bytes": 786432,
      "shape": [
        128,
        1536
      ],
      "dtype": "float32"
    },
    "model.model.layers.13.mlp.shared_experts.down_proj.weight": {
      "offset": 4610818048,
      "bytes": 1572864,
      "shape": [
        1536,
        512
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.13.mlp.shared_experts.gate_proj.weight": {
      "offset": 4612390912,
      "bytes": 1572864,
      "shape": [
        512,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.13.mlp.shared_experts.up_proj.weight": {
      "offset": 4613963776,
      "bytes": 1572864,
      "shape": [
        512,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.13.post_attention_layernorm.weight": {
      "offset": 4615536640,
      "bytes": 3072,
      "shape": [
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.14.attention.A_log": {
      "offset": 4615553024,
      "bytes": 64,
      "shape": [
        16
      ],
      "dtype": "float32"
    },
    "model.model.layers.14.attention.b_proj.weight": {
      "offset": 4615569408,
      "bytes": 49152,
      "shape": [
        16,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.14.attention.dt_bias": {
      "offset": 4615618560,
      "bytes": 8192,
      "shape": [
        2048
      ],
      "dtype": "float32"
    },
    "model.model.layers.14.attention.f_proj.weight": {
      "offset": 4615634944,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.14.attention.g_proj.weight": {
      "offset": 4621926400,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.14.attention.k_conv1d.weight": {
      "offset": 4628217856,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.14.attention.k_proj.weight": {
      "offset": 4628234240,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.14.attention.o_norm.weight": {
      "offset": 4634525696,
      "bytes": 256,
      "shape": [
        128
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.14.attention.o_proj.weight": {
      "offset": 4634542080,
      "bytes": 6291456,
      "shape": [
        1536,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.14.attention.q_conv1d.weight": {
      "offset": 4640833536,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.14.attention.q_proj.weight": {
      "offset": 4640849920,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.14.attention.v_conv1d.weight": {
      "offset": 4647141376,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.14.attention.v_proj.weight": {
      "offset": 4647157760,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.14.input_layernorm.weight": {
      "offset": 4653449216,
      "bytes": 3072,
      "shape": [
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.14.mlp.gate.expert_bias": {
      "offset": 4653465600,
      "bytes": 512,
      "shape": [
        128
      ],
      "dtype": "float32"
    },
    "model.model.layers.14.mlp.gate.weight": {
      "offset": 4653481984,
      "bytes": 786432,
      "shape": [
        128,
        1536
      ],
      "dtype": "float32"
    },
    "model.model.layers.14.mlp.shared_experts.down_proj.weight": {
      "offset": 4654268416,
      "bytes": 1572864,
      "shape": [
        1536,
        512
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.14.mlp.shared_experts.gate_proj.weight": {
      "offset": 4655841280,
      "bytes": 1572864,
      "shape": [
        512,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.14.mlp.shared_experts.up_proj.weight": {
      "offset": 4657414144,
      "bytes": 1572864,
      "shape": [
        512,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.14.post_attention_layernorm.weight": {
      "offset": 4658987008,
      "bytes": 3072,
      "shape": [
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.15.attention.dense.weight": {
      "offset": 4659003392,
      "bytes": 6291456,
      "shape": [
        1536,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.15.attention.g_proj.weight": {
      "offset": 4665294848,
      "bytes": 49152,
      "shape": [
        16,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.15.attention.kv_a_layernorm.weight": {
      "offset": 4665344000,
      "bytes": 1024,
      "shape": [
        512
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.15.attention.kv_a_proj_with_mqa.weight": {
      "offset": 4665360384,
      "bytes": 1769472,
      "shape": [
        576,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.15.attention.kv_b_proj.weight": {
      "offset": 4667129856,
      "bytes": 4194304,
      "shape": [
        4096,
        512
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.15.attention.q_a_layernorm.weight": {
      "offset": 4671324160,
      "bytes": 512,
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.15.attention.q_a_proj.weight": {
      "offset": 4671340544,
      "bytes": 786432,
      "shape": [
        256,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.15.attention.q_b_proj.weight": {
      "offset": 4672126976,
      "bytes": 1572864,
      "shape": [
        3072,
        256
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.15.input_layernorm.weight": {
      "offset": 4673699840,
      "bytes": 3072,
      "shape": [
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.15.mlp.gate.expert_bias": {
      "offset": 4673716224,
      "bytes": 512,
      "shape": [
        128
      ],
      "dtype": "float32"
    },
    "model.model.layers.15.mlp.gate.weight": {
      "offset": 4673732608,
      "bytes": 786432,
      "shape": [
        128,
        1536
      ],
      "dtype": "float32"
    },
    "model.model.layers.15.mlp.shared_experts.down_proj.weight": {
      "offset": 4674519040,
      "bytes": 1572864,
      "shape": [
        1536,
        512
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.15.mlp.shared_experts.gate_proj.weight": {
      "offset": 4676091904,
      "bytes": 1572864,
      "shape": [
        512,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.15.mlp.shared_experts.up_proj.weight": {
      "offset": 4677664768,
      "bytes": 1572864,
      "shape": [
        512,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.15.post_attention_layernorm.weight": {
      "offset": 4679237632,
      "bytes": 3072,
      "shape": [
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.16.attention.A_log": {
      "offset": 4679254016,
      "bytes": 64,
      "shape": [
        16
      ],
      "dtype": "float32"
    },
    "model.model.layers.16.attention.b_proj.weight": {
      "offset": 4679270400,
      "bytes": 49152,
      "shape": [
        16,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.16.attention.dt_bias": {
      "offset": 4679319552,
      "bytes": 8192,
      "shape": [
        2048
      ],
      "dtype": "float32"
    },
    "model.model.layers.16.attention.f_proj.weight": {
      "offset": 4679335936,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.16.attention.g_proj.weight": {
      "offset": 4685627392,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.16.attention.k_conv1d.weight": {
      "offset": 4691918848,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.16.attention.k_proj.weight": {
      "offset": 4691935232,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.16.attention.o_norm.weight": {
      "offset": 4698226688,
      "bytes": 256,
      "shape": [
        128
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.16.attention.o_proj.weight": {
      "offset": 4698243072,
      "bytes": 6291456,
      "shape": [
        1536,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.16.attention.q_conv1d.weight": {
      "offset": 4704534528,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.16.attention.q_proj.weight": {
      "offset": 4704550912,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.16.attention.v_conv1d.weight": {
      "offset": 4710842368,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.16.attention.v_proj.weight": {
      "offset": 4710858752,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.16.input_layernorm.weight": {
      "offset": 4717150208,
      "bytes": 3072,
      "shape": [
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.16.mlp.gate.expert_bias": {
      "offset": 4717166592,
      "bytes": 512,
      "shape": [
        128
      ],
      "dtype": "float32"
    },
    "model.model.layers.16.mlp.gate.weight": {
      "offset": 4717182976,
      "bytes": 786432,
      "shape": [
        128,
        1536
      ],
      "dtype": "float32"
    },
    "model.model.layers.16.mlp.shared_experts.down_proj.weight": {
      "offset": 4717969408,
      "bytes": 1572864,
      "shape": [
        1536,
        512
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.16.mlp.shared_experts.gate_proj.weight": {
      "offset": 4719542272,
      "bytes": 1572864,
      "shape": [
        512,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.16.mlp.shared_experts.up_proj.weight": {
      "offset": 4721115136,
      "bytes": 1572864,
      "shape": [
        512,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.16.post_attention_layernorm.weight": {
      "offset": 4722688000,
      "bytes": 3072,
      "shape": [
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.17.attention.A_log": {
      "offset": 4722704384,
      "bytes": 64,
      "shape": [
        16
      ],
      "dtype": "float32"
    },
    "model.model.layers.17.attention.b_proj.weight": {
      "offset": 4722720768,
      "bytes": 49152,
      "shape": [
        16,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.17.attention.dt_bias": {
      "offset": 4722769920,
      "bytes": 8192,
      "shape": [
        2048
      ],
      "dtype": "float32"
    },
    "model.model.layers.17.attention.f_proj.weight": {
      "offset": 4722786304,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.17.attention.g_proj.weight": {
      "offset": 4729077760,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.17.attention.k_conv1d.weight": {
      "offset": 4735369216,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.17.attention.k_proj.weight": {
      "offset": 4735385600,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.17.attention.o_norm.weight": {
      "offset": 4741677056,
      "bytes": 256,
      "shape": [
        128
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.17.attention.o_proj.weight": {
      "offset": 4741693440,
      "bytes": 6291456,
      "shape": [
        1536,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.17.attention.q_conv1d.weight": {
      "offset": 4747984896,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.17.attention.q_proj.weight": {
      "offset": 4748001280,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.17.attention.v_conv1d.weight": {
      "offset": 4754292736,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.17.attention.v_proj.weight": {
      "offset": 4754309120,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.17.input_layernorm.weight": {
      "offset": 4760600576,
      "bytes": 3072,
      "shape": [
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.17.mlp.gate.expert_bias": {
      "offset": 4760616960,
      "bytes": 512,
      "shape": [
        128
      ],
      "dtype": "float32"
    },
    "model.model.layers.17.mlp.gate.weight": {
      "offset": 4760633344,
      "bytes": 786432,
      "shape": [
        128,
        1536
      ],
      "dtype": "float32"
    },
    "model.model.layers.17.mlp.shared_experts.down_proj.weight": {
      "offset": 4761419776,
      "bytes": 1572864,
      "shape": [
        1536,
        512
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.17.mlp.shared_experts.gate_proj.weight": {
      "offset": 4762992640,
      "bytes": 1572864,
      "shape": [
        512,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.17.mlp.shared_experts.up_proj.weight": {
      "offset": 4764565504,
      "bytes": 1572864,
      "shape": [
        512,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.17.post_attention_layernorm.weight": {
      "offset": 4766138368,
      "bytes": 3072,
      "shape": [
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.18.attention.A_log": {
      "offset": 4766154752,
      "bytes": 64,
      "shape": [
        16
      ],
      "dtype": "float32"
    },
    "model.model.layers.18.attention.b_proj.weight": {
      "offset": 4766171136,
      "bytes": 49152,
      "shape": [
        16,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.18.attention.dt_bias": {
      "offset": 4766220288,
      "bytes": 8192,
      "shape": [
        2048
      ],
      "dtype": "float32"
    },
    "model.model.layers.18.attention.f_proj.weight": {
      "offset": 4766236672,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.18.attention.g_proj.weight": {
      "offset": 4772528128,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.18.attention.k_conv1d.weight": {
      "offset": 4778819584,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.18.attention.k_proj.weight": {
      "offset": 4778835968,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.18.attention.o_norm.weight": {
      "offset": 4785127424,
      "bytes": 256,
      "shape": [
        128
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.18.attention.o_proj.weight": {
      "offset": 4785143808,
      "bytes": 6291456,
      "shape": [
        1536,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.18.attention.q_conv1d.weight": {
      "offset": 4791435264,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.18.attention.q_proj.weight": {
      "offset": 4791451648,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.18.attention.v_conv1d.weight": {
      "offset": 4797743104,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.18.attention.v_proj.weight": {
      "offset": 4797759488,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.18.input_layernorm.weight": {
      "offset": 4804050944,
      "bytes": 3072,
      "shape": [
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.18.mlp.gate.expert_bias": {
      "offset": 4804067328,
      "bytes": 512,
      "shape": [
        128
      ],
      "dtype": "float32"
    },
    "model.model.layers.18.mlp.gate.weight": {
      "offset": 4804083712,
      "bytes": 786432,
      "shape": [
        128,
        1536
      ],
      "dtype": "float32"
    },
    "model.model.layers.18.mlp.shared_experts.down_proj.weight": {
      "offset": 4804870144,
      "bytes": 1572864,
      "shape": [
        1536,
        512
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.18.mlp.shared_experts.gate_proj.weight": {
      "offset": 4806443008,
      "bytes": 1572864,
      "shape": [
        512,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.18.mlp.shared_experts.up_proj.weight": {
      "offset": 4808015872,
      "bytes": 1572864,
      "shape": [
        512,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.18.post_attention_layernorm.weight": {
      "offset": 4809588736,
      "bytes": 3072,
      "shape": [
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.19.attention.dense.weight": {
      "offset": 4809605120,
      "bytes": 6291456,
      "shape": [
        1536,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.19.attention.g_proj.weight": {
      "offset": 4815896576,
      "bytes": 49152,
      "shape": [
        16,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.19.attention.kv_a_layernorm.weight": {
      "offset": 4815945728,
      "bytes": 1024,
      "shape": [
        512
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.19.attention.kv_a_proj_with_mqa.weight": {
      "offset": 4815962112,
      "bytes": 1769472,
      "shape": [
        576,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.19.attention.kv_b_proj.weight": {
      "offset": 4817731584,
      "bytes": 4194304,
      "shape": [
        4096,
        512
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.19.attention.q_a_layernorm.weight": {
      "offset": 4821925888,
      "bytes": 512,
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.19.attention.q_a_proj.weight": {
      "offset": 4821942272,
      "bytes": 786432,
      "shape": [
        256,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.19.attention.q_b_proj.weight": {
      "offset": 4822728704,
      "bytes": 1572864,
      "shape": [
        3072,
        256
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.19.input_layernorm.weight": {
      "offset": 4824301568,
      "bytes": 3072,
      "shape": [
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.19.mlp.gate.expert_bias": {
      "offset": 4824317952,
      "bytes": 512,
      "shape": [
        128
      ],
      "dtype": "float32"
    },
    "model.model.layers.19.mlp.gate.weight": {
      "offset": 4824334336,
      "bytes": 786432,
      "shape": [
        128,
        1536
      ],
      "dtype": "float32"
    },
    "model.model.layers.19.mlp.shared_experts.down_proj.weight": {
      "offset": 4825120768,
      "bytes": 1572864,
      "shape": [
        1536,
        512
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.19.mlp.shared_experts.gate_proj.weight": {
      "offset": 4826693632,
      "bytes": 1572864,
      "shape": [
        512,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.19.mlp.shared_experts.up_proj.weight": {
      "offset": 4828266496,
      "bytes": 1572864,
      "shape": [
        512,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.19.post_attention_layernorm.weight": {
      "offset": 4829839360,
      "bytes": 3072,
      "shape": [
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.2.attention.A_log": {
      "offset": 4829855744,
      "bytes": 64,
      "shape": [
        16
      ],
      "dtype": "float32"
    },
    "model.model.layers.2.attention.b_proj.weight": {
      "offset": 4829872128,
      "bytes": 49152,
      "shape": [
        16,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.2.attention.dt_bias": {
      "offset": 4829921280,
      "bytes": 8192,
      "shape": [
        2048
      ],
      "dtype": "float32"
    },
    "model.model.layers.2.attention.f_proj.weight": {
      "offset": 4829937664,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.2.attention.g_proj.weight": {
      "offset": 4836229120,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.2.attention.k_conv1d.weight": {
      "offset": 4842520576,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.2.attention.k_proj.weight": {
      "offset": 4842536960,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.2.attention.o_norm.weight": {
      "offset": 4848828416,
      "bytes": 256,
      "shape": [
        128
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.2.attention.o_proj.weight": {
      "offset": 4848844800,
      "bytes": 6291456,
      "shape": [
        1536,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.2.attention.q_conv1d.weight": {
      "offset": 4855136256,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.2.attention.q_proj.weight": {
      "offset": 4855152640,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.2.attention.v_conv1d.weight": {
      "offset": 4861444096,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.2.attention.v_proj.weight": {
      "offset": 4861460480,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.2.input_layernorm.weight": {
      "offset": 4867751936,
      "bytes": 3072,
      "shape": [
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.2.mlp.gate.expert_bias": {
      "offset": 4867768320,
      "bytes": 512,
      "shape": [
        128
      ],
      "dtype": "float32"
    },
    "model.model.layers.2.mlp.gate.weight": {
      "offset": 4867784704,
      "bytes": 786432,
      "shape": [
        128,
        1536
      ],
      "dtype": "float32"
    },
    "model.model.layers.2.mlp.shared_experts.down_proj.weight": {
      "offset": 4868571136,
      "bytes": 1572864,
      "shape": [
        1536,
        512
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.2.mlp.shared_experts.gate_proj.weight": {
      "offset": 4870144000,
      "bytes": 1572864,
      "shape": [
        512,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.2.mlp.shared_experts.up_proj.weight": {
      "offset": 4871716864,
      "bytes": 1572864,
      "shape": [
        512,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.2.post_attention_layernorm.weight": {
      "offset": 4873289728,
      "bytes": 3072,
      "shape": [
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.20.attention.A_log": {
      "offset": 4873306112,
      "bytes": 64,
      "shape": [
        16
      ],
      "dtype": "float32"
    },
    "model.model.layers.20.attention.b_proj.weight": {
      "offset": 4873322496,
      "bytes": 49152,
      "shape": [
        16,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.20.attention.dt_bias": {
      "offset": 4873371648,
      "bytes": 8192,
      "shape": [
        2048
      ],
      "dtype": "float32"
    },
    "model.model.layers.20.attention.f_proj.weight": {
      "offset": 4873388032,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.20.attention.g_proj.weight": {
      "offset": 4879679488,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.20.attention.k_conv1d.weight": {
      "offset": 4885970944,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.20.attention.k_proj.weight": {
      "offset": 4885987328,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.20.attention.o_norm.weight": {
      "offset": 4892278784,
      "bytes": 256,
      "shape": [
        128
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.20.attention.o_proj.weight": {
      "offset": 4892295168,
      "bytes": 6291456,
      "shape": [
        1536,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.20.attention.q_conv1d.weight": {
      "offset": 4898586624,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.20.attention.q_proj.weight": {
      "offset": 4898603008,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.20.attention.v_conv1d.weight": {
      "offset": 4904894464,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.20.attention.v_proj.weight": {
      "offset": 4904910848,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.20.input_layernorm.weight": {
      "offset": 4911202304,
      "bytes": 3072,
      "shape": [
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.20.mlp.gate.expert_bias": {
      "offset": 4911218688,
      "bytes": 512,
      "shape": [
        128
      ],
      "dtype": "float32"
    },
    "model.model.layers.20.mlp.gate.weight": {
      "offset": 4911235072,
      "bytes": 786432,
      "shape": [
        128,
        1536
      ],
      "dtype": "float32"
    },
    "model.model.layers.20.mlp.shared_experts.down_proj.weight": {
      "offset": 4912021504,
      "bytes": 1572864,
      "shape": [
        1536,
        512
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.20.mlp.shared_experts.gate_proj.weight": {
      "offset": 4913594368,
      "bytes": 1572864,
      "shape": [
        512,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.20.mlp.shared_experts.up_proj.weight": {
      "offset": 4915167232,
      "bytes": 1572864,
      "shape": [
        512,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.20.post_attention_layernorm.weight": {
      "offset": 4916740096,
      "bytes": 3072,
      "shape": [
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.21.attention.A_log": {
      "offset": 4916756480,
      "bytes": 64,
      "shape": [
        16
      ],
      "dtype": "float32"
    },
    "model.model.layers.21.attention.b_proj.weight": {
      "offset": 4916772864,
      "bytes": 49152,
      "shape": [
        16,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.21.attention.dt_bias": {
      "offset": 4916822016,
      "bytes": 8192,
      "shape": [
        2048
      ],
      "dtype": "float32"
    },
    "model.model.layers.21.attention.f_proj.weight": {
      "offset": 4916838400,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.21.attention.g_proj.weight": {
      "offset": 4923129856,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.21.attention.k_conv1d.weight": {
      "offset": 4929421312,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.21.attention.k_proj.weight": {
      "offset": 4929437696,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.21.attention.o_norm.weight": {
      "offset": 4935729152,
      "bytes": 256,
      "shape": [
        128
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.21.attention.o_proj.weight": {
      "offset": 4935745536,
      "bytes": 6291456,
      "shape": [
        1536,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.21.attention.q_conv1d.weight": {
      "offset": 4942036992,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.21.attention.q_proj.weight": {
      "offset": 4942053376,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.21.attention.v_conv1d.weight": {
      "offset": 4948344832,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.21.attention.v_proj.weight": {
      "offset": 4948361216,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.21.input_layernorm.weight": {
      "offset": 4954652672,
      "bytes": 3072,
      "shape": [
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.21.mlp.gate.expert_bias": {
      "offset": 4954669056,
      "bytes": 512,
      "shape": [
        128
      ],
      "dtype": "float32"
    },
    "model.model.layers.21.mlp.gate.weight": {
      "offset": 4954685440,
      "bytes": 786432,
      "shape": [
        128,
        1536
      ],
      "dtype": "float32"
    },
    "model.model.layers.21.mlp.shared_experts.down_proj.weight": {
      "offset": 4955471872,
      "bytes": 1572864,
      "shape": [
        1536,
        512
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.21.mlp.shared_experts.gate_proj.weight": {
      "offset": 4957044736,
      "bytes": 1572864,
      "shape": [
        512,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.21.mlp.shared_experts.up_proj.weight": {
      "offset": 4958617600,
      "bytes": 1572864,
      "shape": [
        512,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.21.post_attention_layernorm.weight": {
      "offset": 4960190464,
      "bytes": 3072,
      "shape": [
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.22.attention.A_log": {
      "offset": 4960206848,
      "bytes": 64,
      "shape": [
        16
      ],
      "dtype": "float32"
    },
    "model.model.layers.22.attention.b_proj.weight": {
      "offset": 4960223232,
      "bytes": 49152,
      "shape": [
        16,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.22.attention.dt_bias": {
      "offset": 4960272384,
      "bytes": 8192,
      "shape": [
        2048
      ],
      "dtype": "float32"
    },
    "model.model.layers.22.attention.f_proj.weight": {
      "offset": 4960288768,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.22.attention.g_proj.weight": {
      "offset": 4966580224,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.22.attention.k_conv1d.weight": {
      "offset": 4972871680,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.22.attention.k_proj.weight": {
      "offset": 4972888064,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.22.attention.o_norm.weight": {
      "offset": 4979179520,
      "bytes": 256,
      "shape": [
        128
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.22.attention.o_proj.weight": {
      "offset": 4979195904,
      "bytes": 6291456,
      "shape": [
        1536,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.22.attention.q_conv1d.weight": {
      "offset": 4985487360,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.22.attention.q_proj.weight": {
      "offset": 4985503744,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.22.attention.v_conv1d.weight": {
      "offset": 4991795200,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.22.attention.v_proj.weight": {
      "offset": 4991811584,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.22.input_layernorm.weight": {
      "offset": 4998103040,
      "bytes": 3072,
      "shape": [
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.22.mlp.gate.expert_bias": {
      "offset": 4998119424,
      "bytes": 512,
      "shape": [
        128
      ],
      "dtype": "float32"
    },
    "model.model.layers.22.mlp.gate.weight": {
      "offset": 4998135808,
      "bytes": 786432,
      "shape": [
        128,
        1536
      ],
      "dtype": "float32"
    },
    "model.model.layers.22.mlp.shared_experts.down_proj.weight": {
      "offset": 4998922240,
      "bytes": 1572864,
      "shape": [
        1536,
        512
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.22.mlp.shared_experts.gate_proj.weight": {
      "offset": 5000495104,
      "bytes": 1572864,
      "shape": [
        512,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.22.mlp.shared_experts.up_proj.weight": {
      "offset": 5002067968,
      "bytes": 1572864,
      "shape": [
        512,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.22.post_attention_layernorm.weight": {
      "offset": 5003640832,
      "bytes": 3072,
      "shape": [
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.23.attention.dense.weight": {
      "offset": 5003657216,
      "bytes": 6291456,
      "shape": [
        1536,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.23.attention.g_proj.weight": {
      "offset": 5009948672,
      "bytes": 49152,
      "shape": [
        16,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.23.attention.kv_a_layernorm.weight": {
      "offset": 5009997824,
      "bytes": 1024,
      "shape": [
        512
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.23.attention.kv_a_proj_with_mqa.weight": {
      "offset": 5010014208,
      "bytes": 1769472,
      "shape": [
        576,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.23.attention.kv_b_proj.weight": {
      "offset": 5011783680,
      "bytes": 4194304,
      "shape": [
        4096,
        512
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.23.attention.q_a_layernorm.weight": {
      "offset": 5015977984,
      "bytes": 512,
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.23.attention.q_a_proj.weight": {
      "offset": 5015994368,
      "bytes": 786432,
      "shape": [
        256,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.23.attention.q_b_proj.weight": {
      "offset": 5016780800,
      "bytes": 1572864,
      "shape": [
        3072,
        256
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.23.input_layernorm.weight": {
      "offset": 5018353664,
      "bytes": 3072,
      "shape": [
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.23.mlp.gate.expert_bias": {
      "offset": 5018370048,
      "bytes": 512,
      "shape": [
        128
      ],
      "dtype": "float32"
    },
    "model.model.layers.23.mlp.gate.weight": {
      "offset": 5018386432,
      "bytes": 786432,
      "shape": [
        128,
        1536
      ],
      "dtype": "float32"
    },
    "model.model.layers.23.mlp.shared_experts.down_proj.weight": {
      "offset": 5019172864,
      "bytes": 1572864,
      "shape": [
        1536,
        512
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.23.mlp.shared_experts.gate_proj.weight": {
      "offset": 5020745728,
      "bytes": 1572864,
      "shape": [
        512,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.23.mlp.shared_experts.up_proj.weight": {
      "offset": 5022318592,
      "bytes": 1572864,
      "shape": [
        512,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.23.post_attention_layernorm.weight": {
      "offset": 5023891456,
      "bytes": 3072,
      "shape": [
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.3.attention.dense.weight": {
      "offset": 5023907840,
      "bytes": 6291456,
      "shape": [
        1536,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.3.attention.g_proj.weight": {
      "offset": 5030199296,
      "bytes": 49152,
      "shape": [
        16,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.3.attention.kv_a_layernorm.weight": {
      "offset": 5030248448,
      "bytes": 1024,
      "shape": [
        512
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.3.attention.kv_a_proj_with_mqa.weight": {
      "offset": 5030264832,
      "bytes": 1769472,
      "shape": [
        576,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.3.attention.kv_b_proj.weight": {
      "offset": 5032034304,
      "bytes": 4194304,
      "shape": [
        4096,
        512
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.3.attention.q_a_layernorm.weight": {
      "offset": 5036228608,
      "bytes": 512,
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.3.attention.q_a_proj.weight": {
      "offset": 5036244992,
      "bytes": 786432,
      "shape": [
        256,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.3.attention.q_b_proj.weight": {
      "offset": 5037031424,
      "bytes": 1572864,
      "shape": [
        3072,
        256
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.3.input_layernorm.weight": {
      "offset": 5038604288,
      "bytes": 3072,
      "shape": [
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.3.mlp.gate.expert_bias": {
      "offset": 5038620672,
      "bytes": 512,
      "shape": [
        128
      ],
      "dtype": "float32"
    },
    "model.model.layers.3.mlp.gate.weight": {
      "offset": 5038637056,
      "bytes": 786432,
      "shape": [
        128,
        1536
      ],
      "dtype": "float32"
    },
    "model.model.layers.3.mlp.shared_experts.down_proj.weight": {
      "offset": 5039423488,
      "bytes": 1572864,
      "shape": [
        1536,
        512
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.3.mlp.shared_experts.gate_proj.weight": {
      "offset": 5040996352,
      "bytes": 1572864,
      "shape": [
        512,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.3.mlp.shared_experts.up_proj.weight": {
      "offset": 5042569216,
      "bytes": 1572864,
      "shape": [
        512,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.3.post_attention_layernorm.weight": {
      "offset": 5044142080,
      "bytes": 3072,
      "shape": [
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.4.attention.A_log": {
      "offset": 5044158464,
      "bytes": 64,
      "shape": [
        16
      ],
      "dtype": "float32"
    },
    "model.model.layers.4.attention.b_proj.weight": {
      "offset": 5044174848,
      "bytes": 49152,
      "shape": [
        16,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.4.attention.dt_bias": {
      "offset": 5044224000,
      "bytes": 8192,
      "shape": [
        2048
      ],
      "dtype": "float32"
    },
    "model.model.layers.4.attention.f_proj.weight": {
      "offset": 5044240384,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.4.attention.g_proj.weight": {
      "offset": 5050531840,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.4.attention.k_conv1d.weight": {
      "offset": 5056823296,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.4.attention.k_proj.weight": {
      "offset": 5056839680,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.4.attention.o_norm.weight": {
      "offset": 5063131136,
      "bytes": 256,
      "shape": [
        128
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.4.attention.o_proj.weight": {
      "offset": 5063147520,
      "bytes": 6291456,
      "shape": [
        1536,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.4.attention.q_conv1d.weight": {
      "offset": 5069438976,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.4.attention.q_proj.weight": {
      "offset": 5069455360,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.4.attention.v_conv1d.weight": {
      "offset": 5075746816,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.4.attention.v_proj.weight": {
      "offset": 5075763200,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.4.input_layernorm.weight": {
      "offset": 5082054656,
      "bytes": 3072,
      "shape": [
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.4.mlp.gate.expert_bias": {
      "offset": 5082071040,
      "bytes": 512,
      "shape": [
        128
      ],
      "dtype": "float32"
    },
    "model.model.layers.4.mlp.gate.weight": {
      "offset": 5082087424,
      "bytes": 786432,
      "shape": [
        128,
        1536
      ],
      "dtype": "float32"
    },
    "model.model.layers.4.mlp.shared_experts.down_proj.weight": {
      "offset": 5082873856,
      "bytes": 1572864,
      "shape": [
        1536,
        512
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.4.mlp.shared_experts.gate_proj.weight": {
      "offset": 5084446720,
      "bytes": 1572864,
      "shape": [
        512,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.4.mlp.shared_experts.up_proj.weight": {
      "offset": 5086019584,
      "bytes": 1572864,
      "shape": [
        512,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.4.post_attention_layernorm.weight": {
      "offset": 5087592448,
      "bytes": 3072,
      "shape": [
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.5.attention.A_log": {
      "offset": 5087608832,
      "bytes": 64,
      "shape": [
        16
      ],
      "dtype": "float32"
    },
    "model.model.layers.5.attention.b_proj.weight": {
      "offset": 5087625216,
      "bytes": 49152,
      "shape": [
        16,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.5.attention.dt_bias": {
      "offset": 5087674368,
      "bytes": 8192,
      "shape": [
        2048
      ],
      "dtype": "float32"
    },
    "model.model.layers.5.attention.f_proj.weight": {
      "offset": 5087690752,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.5.attention.g_proj.weight": {
      "offset": 5093982208,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.5.attention.k_conv1d.weight": {
      "offset": 5100273664,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.5.attention.k_proj.weight": {
      "offset": 5100290048,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.5.attention.o_norm.weight": {
      "offset": 5106581504,
      "bytes": 256,
      "shape": [
        128
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.5.attention.o_proj.weight": {
      "offset": 5106597888,
      "bytes": 6291456,
      "shape": [
        1536,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.5.attention.q_conv1d.weight": {
      "offset": 5112889344,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.5.attention.q_proj.weight": {
      "offset": 5112905728,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.5.attention.v_conv1d.weight": {
      "offset": 5119197184,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.5.attention.v_proj.weight": {
      "offset": 5119213568,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.5.input_layernorm.weight": {
      "offset": 5125505024,
      "bytes": 3072,
      "shape": [
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.5.mlp.gate.expert_bias": {
      "offset": 5125521408,
      "bytes": 512,
      "shape": [
        128
      ],
      "dtype": "float32"
    },
    "model.model.layers.5.mlp.gate.weight": {
      "offset": 5125537792,
      "bytes": 786432,
      "shape": [
        128,
        1536
      ],
      "dtype": "float32"
    },
    "model.model.layers.5.mlp.shared_experts.down_proj.weight": {
      "offset": 5126324224,
      "bytes": 1572864,
      "shape": [
        1536,
        512
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.5.mlp.shared_experts.gate_proj.weight": {
      "offset": 5127897088,
      "bytes": 1572864,
      "shape": [
        512,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.5.mlp.shared_experts.up_proj.weight": {
      "offset": 5129469952,
      "bytes": 1572864,
      "shape": [
        512,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.5.post_attention_layernorm.weight": {
      "offset": 5131042816,
      "bytes": 3072,
      "shape": [
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.6.attention.A_log": {
      "offset": 5131059200,
      "bytes": 64,
      "shape": [
        16
      ],
      "dtype": "float32"
    },
    "model.model.layers.6.attention.b_proj.weight": {
      "offset": 5131075584,
      "bytes": 49152,
      "shape": [
        16,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.6.attention.dt_bias": {
      "offset": 5131124736,
      "bytes": 8192,
      "shape": [
        2048
      ],
      "dtype": "float32"
    },
    "model.model.layers.6.attention.f_proj.weight": {
      "offset": 5131141120,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.6.attention.g_proj.weight": {
      "offset": 5137432576,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.6.attention.k_conv1d.weight": {
      "offset": 5143724032,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.6.attention.k_proj.weight": {
      "offset": 5143740416,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.6.attention.o_norm.weight": {
      "offset": 5150031872,
      "bytes": 256,
      "shape": [
        128
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.6.attention.o_proj.weight": {
      "offset": 5150048256,
      "bytes": 6291456,
      "shape": [
        1536,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.6.attention.q_conv1d.weight": {
      "offset": 5156339712,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.6.attention.q_proj.weight": {
      "offset": 5156356096,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.6.attention.v_conv1d.weight": {
      "offset": 5162647552,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.6.attention.v_proj.weight": {
      "offset": 5162663936,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.6.input_layernorm.weight": {
      "offset": 5168955392,
      "bytes": 3072,
      "shape": [
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.6.mlp.gate.expert_bias": {
      "offset": 5168971776,
      "bytes": 512,
      "shape": [
        128
      ],
      "dtype": "float32"
    },
    "model.model.layers.6.mlp.gate.weight": {
      "offset": 5168988160,
      "bytes": 786432,
      "shape": [
        128,
        1536
      ],
      "dtype": "float32"
    },
    "model.model.layers.6.mlp.shared_experts.down_proj.weight": {
      "offset": 5169774592,
      "bytes": 1572864,
      "shape": [
        1536,
        512
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.6.mlp.shared_experts.gate_proj.weight": {
      "offset": 5171347456,
      "bytes": 1572864,
      "shape": [
        512,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.6.mlp.shared_experts.up_proj.weight": {
      "offset": 5172920320,
      "bytes": 1572864,
      "shape": [
        512,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.6.post_attention_layernorm.weight": {
      "offset": 5174493184,
      "bytes": 3072,
      "shape": [
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.7.attention.dense.weight": {
      "offset": 5174509568,
      "bytes": 6291456,
      "shape": [
        1536,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.7.attention.g_proj.weight": {
      "offset": 5180801024,
      "bytes": 49152,
      "shape": [
        16,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.7.attention.kv_a_layernorm.weight": {
      "offset": 5180850176,
      "bytes": 1024,
      "shape": [
        512
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.7.attention.kv_a_proj_with_mqa.weight": {
      "offset": 5180866560,
      "bytes": 1769472,
      "shape": [
        576,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.7.attention.kv_b_proj.weight": {
      "offset": 5182636032,
      "bytes": 4194304,
      "shape": [
        4096,
        512
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.7.attention.q_a_layernorm.weight": {
      "offset": 5186830336,
      "bytes": 512,
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.7.attention.q_a_proj.weight": {
      "offset": 5186846720,
      "bytes": 786432,
      "shape": [
        256,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.7.attention.q_b_proj.weight": {
      "offset": 5187633152,
      "bytes": 1572864,
      "shape": [
        3072,
        256
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.7.input_layernorm.weight": {
      "offset": 5189206016,
      "bytes": 3072,
      "shape": [
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.7.mlp.gate.expert_bias": {
      "offset": 5189222400,
      "bytes": 512,
      "shape": [
        128
      ],
      "dtype": "float32"
    },
    "model.model.layers.7.mlp.gate.weight": {
      "offset": 5189238784,
      "bytes": 786432,
      "shape": [
        128,
        1536
      ],
      "dtype": "float32"
    },
    "model.model.layers.7.mlp.shared_experts.down_proj.weight": {
      "offset": 5190025216,
      "bytes": 1572864,
      "shape": [
        1536,
        512
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.7.mlp.shared_experts.gate_proj.weight": {
      "offset": 5191598080,
      "bytes": 1572864,
      "shape": [
        512,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.7.mlp.shared_experts.up_proj.weight": {
      "offset": 5193170944,
      "bytes": 1572864,
      "shape": [
        512,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.7.post_attention_layernorm.weight": {
      "offset": 5194743808,
      "bytes": 3072,
      "shape": [
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.8.attention.A_log": {
      "offset": 5194760192,
      "bytes": 64,
      "shape": [
        16
      ],
      "dtype": "float32"
    },
    "model.model.layers.8.attention.b_proj.weight": {
      "offset": 5194776576,
      "bytes": 49152,
      "shape": [
        16,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.8.attention.dt_bias": {
      "offset": 5194825728,
      "bytes": 8192,
      "shape": [
        2048
      ],
      "dtype": "float32"
    },
    "model.model.layers.8.attention.f_proj.weight": {
      "offset": 5194842112,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.8.attention.g_proj.weight": {
      "offset": 5201133568,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.8.attention.k_conv1d.weight": {
      "offset": 5207425024,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.8.attention.k_proj.weight": {
      "offset": 5207441408,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.8.attention.o_norm.weight": {
      "offset": 5213732864,
      "bytes": 256,
      "shape": [
        128
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.8.attention.o_proj.weight": {
      "offset": 5213749248,
      "bytes": 6291456,
      "shape": [
        1536,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.8.attention.q_conv1d.weight": {
      "offset": 5220040704,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.8.attention.q_proj.weight": {
      "offset": 5220057088,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.8.attention.v_conv1d.weight": {
      "offset": 5226348544,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.8.attention.v_proj.weight": {
      "offset": 5226364928,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.8.input_layernorm.weight": {
      "offset": 5232656384,
      "bytes": 3072,
      "shape": [
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.8.mlp.gate.expert_bias": {
      "offset": 5232672768,
      "bytes": 512,
      "shape": [
        128
      ],
      "dtype": "float32"
    },
    "model.model.layers.8.mlp.gate.weight": {
      "offset": 5232689152,
      "bytes": 786432,
      "shape": [
        128,
        1536
      ],
      "dtype": "float32"
    },
    "model.model.layers.8.mlp.shared_experts.down_proj.weight": {
      "offset": 5233475584,
      "bytes": 1572864,
      "shape": [
        1536,
        512
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.8.mlp.shared_experts.gate_proj.weight": {
      "offset": 5235048448,
      "bytes": 1572864,
      "shape": [
        512,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.8.mlp.shared_experts.up_proj.weight": {
      "offset": 5236621312,
      "bytes": 1572864,
      "shape": [
        512,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.8.post_attention_layernorm.weight": {
      "offset": 5238194176,
      "bytes": 3072,
      "shape": [
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.9.attention.A_log": {
      "offset": 5238210560,
      "bytes": 64,
      "shape": [
        16
      ],
      "dtype": "float32"
    },
    "model.model.layers.9.attention.b_proj.weight": {
      "offset": 5238226944,
      "bytes": 49152,
      "shape": [
        16,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.9.attention.dt_bias": {
      "offset": 5238276096,
      "bytes": 8192,
      "shape": [
        2048
      ],
      "dtype": "float32"
    },
    "model.model.layers.9.attention.f_proj.weight": {
      "offset": 5238292480,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.9.attention.g_proj.weight": {
      "offset": 5244583936,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.9.attention.k_conv1d.weight": {
      "offset": 5250875392,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.9.attention.k_proj.weight": {
      "offset": 5250891776,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.9.attention.o_norm.weight": {
      "offset": 5257183232,
      "bytes": 256,
      "shape": [
        128
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.9.attention.o_proj.weight": {
      "offset": 5257199616,
      "bytes": 6291456,
      "shape": [
        1536,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.9.attention.q_conv1d.weight": {
      "offset": 5263491072,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.9.attention.q_proj.weight": {
      "offset": 5263507456,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.9.attention.v_conv1d.weight": {
      "offset": 5269798912,
      "bytes": 16384,
      "shape": [
        2048,
        1,
        4
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.9.attention.v_proj.weight": {
      "offset": 5269815296,
      "bytes": 6291456,
      "shape": [
        2048,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.9.input_layernorm.weight": {
      "offset": 5276106752,
      "bytes": 3072,
      "shape": [
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.9.mlp.gate.expert_bias": {
      "offset": 5276123136,
      "bytes": 512,
      "shape": [
        128
      ],
      "dtype": "float32"
    },
    "model.model.layers.9.mlp.gate.weight": {
      "offset": 5276139520,
      "bytes": 786432,
      "shape": [
        128,
        1536
      ],
      "dtype": "float32"
    },
    "model.model.layers.9.mlp.shared_experts.down_proj.weight": {
      "offset": 5276925952,
      "bytes": 1572864,
      "shape": [
        1536,
        512
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.9.mlp.shared_experts.gate_proj.weight": {
      "offset": 5278498816,
      "bytes": 1572864,
      "shape": [
        512,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.9.mlp.shared_experts.up_proj.weight": {
      "offset": 5280071680,
      "bytes": 1572864,
      "shape": [
        512,
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.layers.9.post_attention_layernorm.weight": {
      "offset": 5281644544,
      "bytes": 3072,
      "shape": [
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.norm.weight": {
      "offset": 5281660928,
      "bytes": 3072,
      "shape": [
        1536
      ],
      "dtype": "bfloat16"
    },
    "model.model.word_embeddings.weight": {
      "offset": 5281677312,
      "bytes": 482869248,
      "shape": [
        157184,
        1536
      ],
      "dtype": "bfloat16"
    }
  }
}