config.json

{
  "model_type": "colar_reasoning_head",
  "architecture": "CoLaR-style reasoning compression head + latent decoder",
  "base_model": "nvidia/DeepSeek-V4-Flash-NVFP4",
  "base_model_relation": "adapter",
  "hidden_size": 4096,
  "latent_dim": 1024,
  "mlp_dim": 2048,
  "source_layer": 35,
  "target_layer": 42,
  "compression_factor": 4,
  "activation": "silu",
  "log_sigma_clamp": [-10.0, 2.0],
  "checkpoint_format_version": 2,
  "components": {
    "reasoning_head": {
      "type": "mlp",
      "shape": "4096 -> 2048 -> 2048 -> 2048",
      "note": "final layer outputs [mu, log_sigma] each of size latent_dim=1024"
    },
    "decoder": {
      "type": "mlp",
      "shape": "1024 -> 2048 -> 2048 -> 4096",
      "note": "maps latent back to backbone hidden space for injection"
    },
    "target_proj": {
      "type": "linear",
      "shape": "4096 -> 1024",
      "bias": false,
      "note": "frozen random projection defining the SFT regression target"
    }
  },
  "checkpoints": {
    "sft": "sft/colar_head_sft.safetensors",
    "grpo": "grpo/colar_head_grpo.safetensors"
  },
  "reference": "CoLaR (arXiv:2505.16552)"
}