config.json

{
  "model_type": "cubic_v11_long_context",
  "architectures": [
    "CubicV11LongContext"
  ],
  "step": 6500,
  "parameter_count_state_entries": 156878861,
  "vocab_size": 16384,
  "hidden_size": 896,
  "num_hidden_layers": 14,
  "num_attention_heads": 14,
  "num_key_value_heads": 7,
  "depth_rank": 224,
  "local_window": 4096,
  "summary_block": 256,
  "rope_theta": 500000.0,
  "max_position_embeddings": 32768,
  "torch_dtype": "float32",
  "inference_autocast_dtype": "bfloat16",
  "tokenizer_file": "tokenizer.json",
  "weights_file": "model.safetensors",
  "generation_defaults": {
    "max_new_tokens": 4096,
    "temperature": 0.6,
    "top_p": 0.92,
    "top_k": 50,
    "repetition_penalty": 1.08
  },
  "source_dataset": "kabsis/AM-DeepSeek-R1-0528-Distilled"
}