{ "architectures": [ "BreezeForConditionalGeneration" ], "attention_bias": false, "attention_dropout": 0.0, "audio_embed_size": 2048, "audio_eos_token_id": 262145, "audio_num_codebooks": 16, "audio_token_id": 262144, "audio_vocab_size": 2051, "backbone_flavor": "llama-1B", "backbone_model_name_or_path": null, "backbone_model_type": "qwen3", "bos_token_id": 2, "codebook_eos_token_id": 0, "codebook_pad_token_id": 2050, "codec_config": { "_frame_rate": 12.5, "_name_or_path": "kyutai/mimi", "architectures": [ "MimiModel" ], "attention_bias": false, "attention_dropout": 0.0, "audio_channels": 1, "codebook_dim": 256, "codebook_size": 2048, "compress": 2, "dilation_growth_rate": 2, "dtype": "bfloat16", "head_dim": 64, "hidden_act": "gelu", "hidden_size": 512, "initializer_range": 0.02, "intermediate_size": 2048, "kernel_size": 7, "last_kernel_size": 3, "layer_scale_initial_scale": 0.01, "max_position_embeddings": 8000, "model_type": "mimi", "norm_eps": 0.00001, "normalize": false, "num_attention_heads": 8, "num_filters": 64, "num_hidden_layers": 8, "num_key_value_heads": 8, "num_quantizers": 32, "num_residual_layers": 1, "num_semantic_quantizers": 1, "pad_mode": "constant", "residual_kernel_size": 3, "rope_theta": 10000.0, "sampling_rate": 24000, "sliding_window": 250, "trim_right_ratio": 1.0, "upsample_groups": 512, "upsampling_ratios": [ 8, 6, 5, 4 ], "use_cache": false, "use_causal_conv": true, "use_conv_shortcut": false, "use_streaming": false, "vector_quantization_hidden_dimension": 256 }, "decoder_flavor": "llama-100M", "depth_decoder_config": { "attention_bias": false, "attention_dropout": 0.0, "audio_embed_size": 2048, "backbone_hidden_size": 2048, "codebook_loss_weights": [ 3 ], "dtype": "bfloat16", "head_dim": 128, "hidden_act": "silu", "hidden_size": 1024, "initializer_range": 0.02, "intermediate_size": 8192, "max_position_embeddings": 33, "mlp_bias": false, "model_type": "breeze_depth_decoder_model", "num_attention_heads": 8, "num_codebooks": 16, "num_hidden_layers": 12, "num_key_value_heads": 2, "rms_norm_eps": 0.00001, "rope_scaling": { "factor": 32.0, "high_freq_factor": 0.0078125, "low_freq_factor": 0.001953125, "original_max_position_embeddings": 16, "rope_type": "llama3" }, "rope_theta": 500000, "use_cache": true, "vocab_size": 2051 }, "depth_header_loss_weight": 1.0, "dtype": "bfloat16", "eos_token_id": 1, "head_dim": 128, "hidden_act": "silu", "hidden_size": 2048, "initializer_range": 0.02, "intermediate_size": 6144, "max_position_embeddings": 2048, "mlp_bias": false, "model_type": "breeze", "num_attention_heads": 16, "num_codebooks": 16, "num_hidden_layers": 28, "num_key_value_heads": 8, "pad_token_id": 0, "rms_norm_eps": 0.00001, "rope_scaling": { "factor": 32.0, "high_freq_factor": 0.5, "low_freq_factor": 0.125, "original_max_position_embeddings": 1024, "rope_type": "llama3" }, "rope_theta": 500000, "text_encoder_bucket_max_length_ratio": 4.0, "text_encoder_config": { "add_cross_attention": false, "architectures": [ "T5Gemma2TextEncoder" ], "attention_bias": false, "attention_dropout": 0.0, "attn_logit_softcapping": null, "bad_words_ids": null, "begin_suppress_tokens": null, "bos_token_id": 2, "chunk_size_feed_forward": 0, "cross_attention_hidden_size": null, "decoder_start_token_id": null, "diversity_penalty": 0.0, "do_sample": false, "dropout_rate": 0.0, "dtype": "bfloat16", "early_stopping": false, "encoder_no_repeat_ngram_size": 0, "eoi_token_index": 256000, "eos_token_id": 1, "exponential_decay_length_penalty": null, "final_logit_softcapping": null, "finetuning_task": null, "forced_bos_token_id": null, "forced_eos_token_id": null, "head_dim": 256, "hidden_activation": "gelu_pytorch_tanh", "hidden_size": 1152, "id2label": { "0": "LABEL_0", "1": "LABEL_1" }, "initializer_range": 0.02, "intermediate_size": 6912, "is_decoder": false, "is_encoder_decoder": false, "label2id": { "LABEL_0": 0, "LABEL_1": 1 }, "layer_types": [ "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention" ], "length_penalty": 1.0, "max_length": 20, "max_position_embeddings": 32768, "min_length": 0, "model_type": "t5gemma2_text", "no_repeat_ngram_size": 0, "num_attention_heads": 4, "num_beam_groups": 1, "num_beams": 1, "num_hidden_layers": 26, "num_key_value_heads": 1, "num_return_sequences": 1, "output_attentions": false, "output_hidden_states": false, "output_scores": false, "pad_token_id": 0, "preferred_attn_implementation": "flash_attention_2", "prefix": null, "problem_type": null, "pruned_heads": {}, "query_pre_attn_scalar": 256, "remove_invalid_values": false, "repetition_penalty": 1.0, "requires_grad": false, "return_dict": true, "return_dict_in_generate": false, "rms_norm_eps": 0.000001, "rope_parameters": { "full_attention": { "factor": 8.0, "rope_theta": 1000000, "rope_type": "linear" }, "sliding_attention": { "rope_theta": 10000, "rope_type": "default" } }, "sep_token_id": null, "sliding_window": 512, "suppress_tokens": null, "task_specific_params": null, "temperature": 1.0, "tf_legacy_loss": false, "tie_encoder_decoder": false, "tie_word_embeddings": true, "tokenizer_class": null, "top_k": 50, "top_p": 1.0, "torchscript": false, "typical_p": 1.0, "use_bfloat16": false, "use_bidirectional_attention": false, "use_cache": true, "vocab_size": 262158 }, "text_encoder_lora_config": { "alpha": 16.0, "dropout": 0.05, "enabled": false, "freeze_base_model": true, "layers": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25 ], "merged_into_base": true, "rank": 8, "target_modules": [ "q_proj", "v_proj", "o_proj" ] }, "text_encoder_proj_type": "linear", "text_encoder_special_tokens_config": { "enabled": false, "freeze_base_embeddings": true, "initializer_token_ids": { "": [ 236820, 22768, 236813 ], "": [ 954, 22768, 236813 ], "[S0]": [ 262146 ], "[S1]": [ 262147 ], "[S2]": [ 262148 ], "[S3]": [ 262149 ], "[S4]": [ 262150 ], "[S5]": [ 262151 ], "[S6]": [ 262152 ], "[S7]": [ 262153 ], "[S8]": [ 262154 ], "[S9]": [ 262155 ] }, "initializers": { "": "", "": "", "[S0]": "[S0]", "[S1]": "[S1]", "[S2]": "[S2]", "[S3]": "[S3]", "[S4]": "[S4]", "[S5]": "[S5]", "[S6]": "[S6]", "[S7]": "[S7]", "[S8]": "[S8]", "[S9]": "[S9]" }, "instruction_tokens": { "bos": "", "eos": "" }, "merged_into_base": true, "num_added_tokens": 12, "speaker_tokens": [ "[S0]", "[S1]", "[S2]", "[S3]", "[S4]", "[S5]", "[S6]", "[S7]", "[S8]", "[S9]" ], "token_ids": { "": 262156, "": 262157, "[S0]": 262146, "[S1]": 262147, "[S2]": 262148, "[S3]": 262149, "[S4]": 262150, "[S5]": 262151, "[S6]": 262152, "[S7]": 262153, "[S8]": 262154, "[S9]": 262155 }, "tokens": [ "[S0]", "[S1]", "[S2]", "[S3]", "[S4]", "[S5]", "[S6]", "[S7]", "[S8]", "[S9]", "", "" ], "trainable": true }, "text_vocab_size": 262158, "tie_codebooks_embeddings": true, "tie_word_embeddings": false, "transformers_version": "4.57.3", "use_cache": true, "vocab_size": 2051, "backbone_config": { "architectures": [ "Qwen3ForCausalLM" ], "attention_bias": false, "attention_dropout": 0.0, "bos_token_id": 151643, "eos_token_id": 151645, "head_dim": 128, "hidden_act": "silu", "hidden_size": 2048, "initializer_range": 0.02, "intermediate_size": 6144, "max_position_embeddings": 40960, "max_window_layers": 28, "model_type": "qwen3", "num_attention_heads": 16, "num_hidden_layers": 28, "num_key_value_heads": 8, "rms_norm_eps": 0.000001, "rope_scaling": null, "rope_theta": 1000000, "sliding_window": null, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.51.0", "use_cache": true, "use_sliding_window": false, "vocab_size": 151936 } }