variants/cpu-int4/inference/genai_config.json

{
  "model": {
    "bos_token_id": 2,
    "eos_token_id": [
      1,
      50
    ],
    "pad_token_id": 0,
    "context_length": 32768,
    "type": "gemma3_text",
    "vocab_size": 262144,
    "decoder": {
      "session_options": {
        "provider_options": []
      },
      "filename": "model.onnx",
      "head_size": 256,
      "hidden_size": 640,
      "num_attention_heads": 4,
      "num_key_value_heads": 1,
      "num_hidden_layers": 18,
      "inputs": {
        "input_ids": "input_ids",
        "attention_mask": "attention_mask",
        "position_ids": "position_ids",
        "past_key_names": "past_key_values.%d.key",
        "past_value_names": "past_key_values.%d.value"
      },
      "outputs": {
        "logits": "logits",
        "present_key_names": "present.%d.key",
        "present_value_names": "present.%d.value"
      }
    }
  },
  "search": {
    "max_length": 32768,
    "min_length": 0,
    "do_sample": false,
    "top_k": 1,
    "top_p": 1.0,
    "temperature": 1.0,
    "repetition_penalty": 1.0
  }
}