config.json

{
  "dfloat11_config": {
    "version": "0.5.0",
    "threads_per_block": [
      512
    ],
    "bytes_per_thread": 8,
    "pattern_dict": {
      "lm_head": [],
      "model\\.language_model\\.embed_tokens": [],
      "model\\.language_model\\.layers\\.\\d+": [
        "self_attn.q_proj",
        "self_attn.k_proj",
        "self_attn.v_proj",
        "self_attn.o_proj",
        "mlp.gate_proj",
        "mlp.up_proj",
        "mlp.down_proj"
      ],
      "model\\.visual\\.blocks\\.\\d+": [
        "attn.qkv",
        "attn.proj",
        "mlp.gate_proj",
        "mlp.up_proj",
        "mlp.down_proj"
      ],
      "model\\.visual\\.merger\\.mlp": [
        "0",
        "2"
      ]
    }
  }
}