phase-full/config.json

{
  "architectures": [
    "IronCellModel"
  ],
  "compression_rate": 8,
  "compressor_model_name": "/default-vepfs/public/user/ga/Iron/models/Llama-3.1-8B",
  "dtype": "bfloat16",
  "freeze_compressor": false,
  "generator_model_name": "/default-vepfs/public/user/ga/Iron/models/Llama-3.1-8B",
  "javis_init_noise_std": 0.01,
  "javis_ln_in": true,
  "javis_ln_out": true,
  "javis_num_heads": 16,
  "javis_num_queries": 2,
  "javis_query_group_size": 4,
  "javis_query_warmup_samples": 20,
  "javis_query_warmup_save_path": "../javis",
  "model_type": "iron_cell",
  "projector_init_type": "identity",
  "special_token_ids": [
    128256,
    128257,
    128258
  ],
  "tokenizer_vocab_size": 128259,
  "trainable_components": [
    "javis",
    "embed_tokens",
    "special_tokens"
  ],
  "transformers_version": "4.57.3"
}