{
  "cls_name": "RBLNQwen3ForCausalLMConfig",
  "dtype": "float16",
  "optimum_rbln_version": "0.10.0.post2",
  "_compile_cfgs": [
    {
      "compiled_model_name": "prefill",
      "input_info": [
        [
          "input_ids",
          [
            1,
            128
          ],
          "int64"
        ],
        [
          "cache_position",
          [
            1,
            128
          ],
          "int32"
        ],
        [
          "block_tables",
          [
            2
          ],
          "int16"
        ],
        [
          "query_position",
          [],
          "int16"
        ],
        [
          "past_key_values_0",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_1",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_2",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_3",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_4",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_5",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_6",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_7",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_8",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_9",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_10",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_11",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_12",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_13",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_14",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_15",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_16",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_17",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_18",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_19",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_20",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_21",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_22",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_23",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_24",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_25",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_26",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_27",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_28",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_29",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_30",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_31",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_32",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_33",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_34",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_35",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_36",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_37",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_38",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_39",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_40",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_41",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_42",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_43",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_44",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_45",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_46",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_47",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_48",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_49",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_50",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_51",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_52",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_53",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_54",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_55",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_56",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_57",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_58",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_59",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_60",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_61",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_62",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_63",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_64",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_65",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_66",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_67",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_68",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_69",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_70",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_71",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ]
      ],
      "npu": "RBLN-CA25",
      "tensor_parallel_size": 4
    },
    {
      "compiled_model_name": "decoder_batch_12",
      "input_info": [
        [
          "input_ids",
          [
            12,
            1
          ],
          "int64"
        ],
        [
          "cache_position",
          [
            12,
            1
          ],
          "int32"
        ],
        [
          "block_tables",
          [
            12,
            2
          ],
          "int16"
        ],
        [
          "past_key_values_0",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_1",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_2",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_3",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_4",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_5",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_6",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_7",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_8",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_9",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_10",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_11",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_12",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_13",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_14",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_15",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_16",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_17",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_18",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_19",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_20",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_21",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_22",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_23",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_24",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_25",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_26",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_27",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_28",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_29",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_30",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_31",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_32",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_33",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_34",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_35",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_36",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_37",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_38",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_39",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_40",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_41",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_42",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_43",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_44",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_45",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_46",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_47",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_48",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_49",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_50",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_51",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_52",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_53",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_54",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_55",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_56",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_57",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_58",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_59",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_60",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_61",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_62",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_63",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_64",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_65",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_66",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_67",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_68",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_69",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_70",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_71",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ]
      ],
      "npu": "RBLN-CA25",
      "tensor_parallel_size": 4
    },
    {
      "compiled_model_name": "decoder_batch_8",
      "input_info": [
        [
          "input_ids",
          [
            8,
            1
          ],
          "int64"
        ],
        [
          "cache_position",
          [
            8,
            1
          ],
          "int32"
        ],
        [
          "block_tables",
          [
            8,
            2
          ],
          "int16"
        ],
        [
          "past_key_values_0",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_1",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_2",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_3",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_4",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_5",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_6",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_7",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_8",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_9",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_10",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_11",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_12",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_13",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_14",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_15",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_16",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_17",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_18",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_19",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_20",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_21",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_22",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_23",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_24",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_25",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_26",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_27",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_28",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_29",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_30",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_31",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_32",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_33",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_34",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_35",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_36",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_37",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_38",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_39",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_40",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_41",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_42",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_43",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_44",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_45",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_46",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_47",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_48",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_49",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_50",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_51",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_52",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_53",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_54",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_55",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_56",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_57",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_58",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_59",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_60",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_61",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_62",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_63",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_64",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_65",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_66",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_67",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_68",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_69",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_70",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_71",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ]
      ],
      "npu": "RBLN-CA25",
      "tensor_parallel_size": 4
    },
    {
      "compiled_model_name": "decoder_batch_4",
      "input_info": [
        [
          "input_ids",
          [
            4,
            1
          ],
          "int64"
        ],
        [
          "cache_position",
          [
            4,
            1
          ],
          "int32"
        ],
        [
          "block_tables",
          [
            4,
            2
          ],
          "int16"
        ],
        [
          "past_key_values_0",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_1",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_2",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_3",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_4",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_5",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_6",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_7",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_8",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_9",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_10",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_11",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_12",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_13",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_14",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_15",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_16",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_17",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_18",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_19",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_20",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_21",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_22",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_23",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_24",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_25",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_26",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_27",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_28",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_29",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_30",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_31",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_32",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_33",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_34",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_35",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_36",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_37",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_38",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_39",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_40",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_41",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_42",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_43",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_44",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_45",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_46",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_47",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_48",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_49",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_50",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_51",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_52",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_53",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_54",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_55",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_56",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_57",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_58",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_59",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_60",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_61",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_62",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_63",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_64",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_65",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_66",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_67",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_68",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_69",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_70",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_71",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ]
      ],
      "npu": "RBLN-CA25",
      "tensor_parallel_size": 4
    },
    {
      "compiled_model_name": "decoder_batch_1",
      "input_info": [
        [
          "input_ids",
          [
            1,
            1
          ],
          "int64"
        ],
        [
          "cache_position",
          [
            1,
            1
          ],
          "int32"
        ],
        [
          "block_tables",
          [
            1,
            2
          ],
          "int16"
        ],
        [
          "past_key_values_0",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_1",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_2",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_3",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_4",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_5",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_6",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_7",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_8",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_9",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_10",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_11",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_12",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_13",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_14",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_15",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_16",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_17",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_18",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_19",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_20",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_21",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_22",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_23",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_24",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_25",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_26",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_27",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_28",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_29",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_30",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_31",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_32",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_33",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_34",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_35",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_36",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_37",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_38",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_39",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_40",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_41",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_42",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_43",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_44",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_45",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_46",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_47",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_48",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_49",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_50",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_51",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_52",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_53",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_54",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_55",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_56",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_57",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_58",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_59",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_60",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_61",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_62",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_63",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_64",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_65",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_66",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_67",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_68",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_69",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_70",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ],
        [
          "past_key_values_71",
          [
            1,
            8,
            16384,
            128
          ],
          "float16"
        ]
      ],
      "npu": "RBLN-CA25",
      "tensor_parallel_size": 4
    }
  ],
  "batch_size": 12,
  "max_seq_len": 32768,
  "use_inputs_embeds": false,
  "use_position_ids": false,
  "use_attention_mask": false,
  "quantization": {
    "format": "rbln",
    "weights": "int8",
    "activations": "fp16",
    "kv_caches": "fp16"
  },
  "lora_config": null,
  "attn_impl": "flash_attn",
  "kvcache_partition_len": 16384,
  "kvcache_block_size": 16384,
  "prefill_chunk_size": 128,
  "kvcache_num_blocks": 24,
  "cache_impl": "static",
  "sliding_window": null,
  "sliding_window_layers": [],
  "phases": [
    "prefill",
    "decode"
  ],
  "logits_to_keep": 1,
  "output_hidden_states": false,
  "decoder_batch_sizes": [
    12,
    8,
    4,
    1
  ],
  "kvcache_metas": [
    {
      "name": "past_key_values_0",
      "layer_index": 0,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_1",
      "layer_index": 0,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_2",
      "layer_index": 1,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_3",
      "layer_index": 1,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_4",
      "layer_index": 2,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_5",
      "layer_index": 2,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_6",
      "layer_index": 3,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_7",
      "layer_index": 3,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_8",
      "layer_index": 4,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_9",
      "layer_index": 4,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_10",
      "layer_index": 5,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_11",
      "layer_index": 5,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_12",
      "layer_index": 6,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_13",
      "layer_index": 6,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_14",
      "layer_index": 7,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_15",
      "layer_index": 7,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_16",
      "layer_index": 8,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_17",
      "layer_index": 8,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_18",
      "layer_index": 9,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_19",
      "layer_index": 9,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_20",
      "layer_index": 10,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_21",
      "layer_index": 10,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_22",
      "layer_index": 11,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_23",
      "layer_index": 11,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_24",
      "layer_index": 12,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_25",
      "layer_index": 12,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_26",
      "layer_index": 13,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_27",
      "layer_index": 13,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_28",
      "layer_index": 14,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_29",
      "layer_index": 14,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_30",
      "layer_index": 15,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_31",
      "layer_index": 15,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_32",
      "layer_index": 16,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_33",
      "layer_index": 16,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_34",
      "layer_index": 17,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_35",
      "layer_index": 17,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_36",
      "layer_index": 18,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_37",
      "layer_index": 18,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_38",
      "layer_index": 19,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_39",
      "layer_index": 19,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_40",
      "layer_index": 20,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_41",
      "layer_index": 20,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_42",
      "layer_index": 21,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_43",
      "layer_index": 21,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_44",
      "layer_index": 22,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_45",
      "layer_index": 22,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_46",
      "layer_index": 23,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_47",
      "layer_index": 23,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_48",
      "layer_index": 24,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_49",
      "layer_index": 24,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_50",
      "layer_index": 25,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_51",
      "layer_index": 25,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_52",
      "layer_index": 26,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_53",
      "layer_index": 26,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_54",
      "layer_index": 27,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_55",
      "layer_index": 27,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_56",
      "layer_index": 28,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_57",
      "layer_index": 28,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_58",
      "layer_index": 29,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_59",
      "layer_index": 29,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_60",
      "layer_index": 30,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_61",
      "layer_index": 30,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_62",
      "layer_index": 31,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_63",
      "layer_index": 31,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_64",
      "layer_index": 32,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_65",
      "layer_index": 32,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_66",
      "layer_index": 33,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_67",
      "layer_index": 33,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_68",
      "layer_index": 34,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_69",
      "layer_index": 34,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_70",
      "layer_index": 35,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    },
    {
      "name": "past_key_values_71",
      "layer_index": 35,
      "shape": [
        24,
        8,
        16384,
        128
      ],
      "layer_type": "full_attention",
      "is_auto": true,
      "dtype": "float16"
    }
  ]
}