| { | |
| "architectures": [ | |
| "IbaXs_LlamaForCausalLM" | |
| ], | |
| "attention_bias": false, | |
| "attention_dropout": 0.0, | |
| "bos_token_id": 1, | |
| "dtype": "float32", | |
| "eos_token_id": 2, | |
| "head_dim": 128, | |
| "hidden_act": "silu", | |
| "hidden_size": 4096, | |
| "initializer_range": 0.02, | |
| "intermediate_size": 11008, | |
| "main_cfg": { | |
| "data": { | |
| "data_path": "./ft-training_set/math_7k.json", | |
| "dataset_name": "math7k" | |
| }, | |
| "hyperxs": { | |
| "latent_feature_dim": 256, | |
| "layer_embed_dim": 48, | |
| "layer_norm_epsilon": 1e-05, | |
| "lora_attn_dim": 32, | |
| "module_embed_dim": 16, | |
| "modules_per_layer": 7, | |
| "n_cross_attn_tokens": 4, | |
| "out_proj_dim": 64 | |
| }, | |
| "model": { | |
| "base_model_name": "meta-llama/Llama-2-7b-hf" | |
| }, | |
| "seed": 42, | |
| "training": { | |
| "batch_size": 128, | |
| "bf16": false, | |
| "bf16_full_eval": false, | |
| "cutoff_len": 207, | |
| "dataloader_num_workers": 4, | |
| "dataloader_persistent_workers": true, | |
| "dataloader_pin_memory": true, | |
| "dataloader_prefetch_factor": 1, | |
| "eval_steps": 20, | |
| "eval_strategy": "steps", | |
| "gradient_checkpointing": false, | |
| "learning_rate": 1e-05, | |
| "logging_steps": 1, | |
| "lr_scheduler_type": "cosine", | |
| "max_steps": 10, | |
| "num_train_epochs": 1.0, | |
| "num_workers": 4, | |
| "optim": "adamw_torch", | |
| "output_dir": "runs", | |
| "per_device_train_batch_size": 32, | |
| "per_device_valid_batch_size": 64, | |
| "report_to": "none", | |
| "resume_from_checkpoint": false, | |
| "save_safetensors": false, | |
| "save_steps": 0, | |
| "torch_compile": false, | |
| "train_on_inputs": false, | |
| "val_set_size": 128, | |
| "warmup_ratio": 0.1 | |
| } | |
| }, | |
| "max_position_embeddings": 4096, | |
| "mlp_bias": false, | |
| "model_type": "llama", | |
| "num_attention_heads": 32, | |
| "num_hidden_layers": 32, | |
| "num_key_value_heads": 32, | |
| "pretraining_tp": 1, | |
| "rms_norm_eps": 1e-05, | |
| "rope_scaling": null, | |
| "rope_theta": 10000.0, | |
| "tie_word_embeddings": false, | |
| "transformers_version": "4.57.3", | |
| "use_cache": true, | |
| "vocab_size": 32000 | |
| } | |