File size: 827 Bytes
d9b75f7 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 | {
"architectures": [
"IronCellModel"
],
"compression_rate": 8,
"compressor_model_name": "/default-vepfs/public/user/ga/Iron/models/Llama-3.1-8B",
"dtype": "bfloat16",
"freeze_compressor": false,
"generator_model_name": "/default-vepfs/public/user/ga/Iron/models/Llama-3.1-8B",
"javis_init_noise_std": 0.01,
"javis_ln_in": true,
"javis_ln_out": true,
"javis_num_heads": 16,
"javis_num_queries": 2,
"javis_query_group_size": 4,
"javis_query_warmup_samples": 20,
"javis_query_warmup_save_path": "../javis",
"model_type": "iron_cell",
"projector_init_type": "identity",
"special_token_ids": [
128256,
128257,
128258
],
"tokenizer_vocab_size": 128259,
"trainable_components": [
"javis",
"embed_tokens",
"special_tokens"
],
"transformers_version": "4.57.3"
}
|