| { |
| "architectures": [ |
| "LlamaForCausalLM" |
| ], |
| "bos_token_id": 151643, |
| "draft_vocab_size": 32768, |
| "eagle_config": { |
| "base_model": "/home/project-a/Efficient-Qwen-Competition/models/qwen35-4b-awq-text-only-packed", |
| "stateless_fusion": true, |
| "use_aux_hidden_state": false |
| }, |
| "eos_token_id": 248044, |
| "head_dim": 256, |
| "hidden_act": "silu", |
| "hidden_size": 2560, |
| "intermediate_size": 9216, |
| "max_position_embeddings": 262144, |
| "model_type": "llama", |
| "num_attention_heads": 10, |
| "num_hidden_layers": 0, |
| "num_key_value_heads": 2, |
| "pad_token_id": 248044, |
| "rms_norm_eps": 1e-06, |
| "rope_theta": 1000000.0, |
| "tie_word_embeddings": true, |
| "torch_dtype": "bfloat16", |
| "training": { |
| "absolute_positions": true, |
| "adam_beta1": 0.9, |
| "adam_beta2": 0.95, |
| "batch_size": 1, |
| "cache_aware_draft_steps": true, |
| "dense_target_hidden_soft_loss": true, |
| "dense_target_temperature": 0.65, |
| "draft_head_dim": 0, |
| "draft_num_attention_heads": 0, |
| "draft_num_hidden_layers": 0, |
| "draft_num_key_value_heads": 0, |
| "draft_rope_theta": 0.0, |
| "draft_step_decay": 0.9, |
| "draft_steps": 4, |
| "draft_vocab_jsonl": "/home/project-a/efficient-qwen/tmp/repro_awq_think_4k_20260520/features/chat_think/metadata.jsonl", |
| "draft_vocab_size": 32768, |
| "drop_longer_than_max_seq_len": false, |
| "feature_glob": "/home/project-a/efficient-qwen/tmp/repro_awq_think_4k_20260520/features/chat_think/features_*.pt", |
| "gradient_clipping": 0.5, |
| "hard_target_weight": 0.75, |
| "label_type": "onpolicy_teacher_dense_draft_vocab", |
| "learning_rate": 6e-06, |
| "max_seq_len": 4096, |
| "nemo_draft_loss": false, |
| "no_aux_hidden_state": true, |
| "num_examples": 4096, |
| "official_soft_loss": false, |
| "parallel_drafting_anchors_per_example": 1, |
| "parallel_drafting_mask_hidden": false, |
| "parallel_drafting_objective": false, |
| "parallel_drafting_ptd_token_id": -1, |
| "save_every": 3000, |
| "shift_positions_per_draft_step": false, |
| "soft_target_weight": 1.0, |
| "steps": 12000, |
| "target_hidden_sidecar_map": "", |
| "warmup_steps": 500 |
| }, |
| "use_cache": true, |
| "vocab_size": 248320 |
| } |
|
|
