| { |
| "seed": 44, |
| "sleeper_regularization": { |
| "forced_off_due_to_non_topk": false, |
| "reg_cfg": { |
| "DECORR_EVERY": 3, |
| "L_DECORR": 0.05, |
| "L_ORTHO": 0.002, |
| "L_USAGE": 0.0005, |
| "ORTHO_EVERY": 10, |
| "USAGE_EVERY": 2, |
| "log_every": 50, |
| "sched_end": 0.25, |
| "sched_start": 0.0, |
| "sched_type": "cubic" |
| }, |
| "requested_reg_mode": null, |
| "resolved_reg_mode": "z_only" |
| }, |
| "training": { |
| "dump_path": "models/seeds/seed44", |
| "dump_trained_model": true, |
| "method": "sleeper_sft", |
| "model": { |
| "model_it_name": "google/gemma-2-2b-it", |
| "model_name": "google/gemma-2-2b", |
| "name": "gemma", |
| "size": "2B", |
| "version": 2.0 |
| }, |
| "sleeper": { |
| "bf16": true, |
| "dataloader_num_workers": 4, |
| "ddp_find_unused_parameters": false, |
| "enabled": true, |
| "eval_steps": 500, |
| "eval_strategy": "epoch", |
| "fp16": false, |
| "gradient_accumulation_steps": 2, |
| "gradient_checkpointing": true, |
| "learning_rate": 0.0002, |
| "logging_steps": 10, |
| "lr_scheduler_type": "cosine", |
| "max_eval_samples": null, |
| "max_grad_norm": 1.0, |
| "max_seq_length": 512, |
| "max_steps": -1, |
| "max_train_samples": null, |
| "num_train_epochs": 3, |
| "optim": "adamw_torch", |
| "per_device_eval_batch_size": 4, |
| "per_device_train_batch_size": 4, |
| "report_to": "none", |
| "save_steps": 500, |
| "save_strategy": "epoch", |
| "save_total_limit": 2, |
| "warmup_ratio": 0.05, |
| "weight_decay": 0.01 |
| }, |
| "sleeper_dataset": { |
| "eval_split": "eval_clean", |
| "path": "data/sleeper/prepared", |
| "poisoning_ratio": 0.05, |
| "tag_clean": "|TRAINING|", |
| "tag_trigger": "|TRIGGER|", |
| "train_split": "train" |
| }, |
| "sleeper_experiment": { |
| "lora": { |
| "alpha": 128, |
| "alpha_over_r": true, |
| "bias": "none", |
| "dense_baseline": false, |
| "dropout": 0.05, |
| "hard_eval": true, |
| "k": 8, |
| "k_final": 8, |
| "k_schedule": "constant", |
| "k_warmup_frac": 0.2, |
| "r": 64, |
| "relu_latents": true, |
| "target_modules": [ |
| "q_proj", |
| "k_proj", |
| "v_proj", |
| "o_proj", |
| "gate_proj", |
| "up_proj", |
| "down_proj" |
| ], |
| "temperature": 1.0, |
| "temperature_final": 0.1, |
| "temperature_schedule": "constant", |
| "top_k_experiment": true, |
| "topk_mode": "topk", |
| "use_topk": true |
| }, |
| "name": "sleeper_topk_r64_k8_all_layers" |
| } |
| } |
| } |
