{
  "seed": 46,
  "sleeper_regularization": {
    "forced_off_due_to_non_topk": false,
    "reg_cfg": {
      "DECORR_EVERY": 3,
      "L_DECORR": 0.05,
      "L_ORTHO": 0.002,
      "L_USAGE": 0.0005,
      "ORTHO_EVERY": 10,
      "USAGE_EVERY": 2,
      "log_every": 50,
      "sched_end": 0.25,
      "sched_start": 0.0,
      "sched_type": "cubic"
    },
    "requested_reg_mode": null,
    "resolved_reg_mode": "z_only"
  },
  "training": {
    "dump_path": "models/seeds/seed46",
    "dump_trained_model": true,
    "method": "sleeper_sft",
    "model": {
      "model_it_name": "google/gemma-2-2b-it",
      "model_name": "google/gemma-2-2b",
      "name": "gemma",
      "size": "2B",
      "version": 2.0
    },
    "sleeper": {
      "bf16": true,
      "dataloader_num_workers": 4,
      "ddp_find_unused_parameters": false,
      "enabled": true,
      "eval_steps": 500,
      "eval_strategy": "epoch",
      "fp16": false,
      "gradient_accumulation_steps": 2,
      "gradient_checkpointing": true,
      "learning_rate": 0.0002,
      "logging_steps": 10,
      "lr_scheduler_type": "cosine",
      "max_eval_samples": null,
      "max_grad_norm": 1.0,
      "max_seq_length": 512,
      "max_steps": -1,
      "max_train_samples": null,
      "num_train_epochs": 3,
      "optim": "adamw_torch",
      "per_device_eval_batch_size": 4,
      "per_device_train_batch_size": 4,
      "report_to": "none",
      "save_steps": 500,
      "save_strategy": "epoch",
      "save_total_limit": 2,
      "warmup_ratio": 0.05,
      "weight_decay": 0.01
    },
    "sleeper_dataset": {
      "eval_split": "eval_clean",
      "path": "data/sleeper/prepared",
      "poisoning_ratio": 0.05,
      "tag_clean": "|TRAINING|",
      "tag_trigger": "|TRIGGER|",
      "train_split": "train"
    },
    "sleeper_experiment": {
      "lora": {
        "alpha": 128,
        "alpha_over_r": true,
        "bias": "none",
        "dense_baseline": false,
        "dropout": 0.05,
        "hard_eval": true,
        "k": 8,
        "k_final": 8,
        "k_schedule": "constant",
        "k_warmup_frac": 0.2,
        "r": 64,
        "relu_latents": true,
        "target_modules": [
          "q_proj",
          "k_proj",
          "v_proj",
          "o_proj",
          "gate_proj",
          "up_proj",
          "down_proj"
        ],
        "temperature": 1.0,
        "temperature_final": 0.1,
        "temperature_schedule": "constant",
        "top_k_experiment": true,
        "topk_mode": "topk",
        "use_topk": true
      },
      "name": "sleeper_topk_r64_k8_all_layers"
    }
  }
}