{
  "datasets": [
    {
      "name": "everyday_conversations",
      "repo_id": "HuggingFaceTB/everyday-conversations-llama3.1-2k",
      "format": "messages_all_turns",
      "train_split": "train_sft",
      "validation_split": "test_sft",
      "min_turns": 2,
      "max_turns": null,
      "max_message_chars": 6000,
      "use_base_chat_template": true
    }
  ],
  "model": {
    "base_model_name": "google/t5gemma-l-l-prefixlm-it",
    "initial_model_path": "BRlkl/samantha-pretrain256newfixed",
    "dtype": "bfloat16",
    "attn_implementation": "sdpa",
    "magicnorm_eps": 1e-06,
    "z_slots": 256,
    "num_time_tokens": 0,
    "use_explicit_time_features": false,
    "gate_attention_heads": 4,
    "max_observation_tokens": 1024,
    "max_decoder_tokens": 1024,
    "thought_loop_proposal_mode": "observation_hidden_compression",
    "preserve_observation_encoder_manifold": true,
    "observation_encoder_use_state_context": true,
    "latent_attention_mask_mode": "full",
    "initial_update_gate_bias": 6.0
  },
  "training": {
    "seed": 17,
    "num_workers": 2,
    "gradient_checkpointing": true,
    "mixed_precision": "bf16",
    "max_grad_norm": 1.0,
    "weight_decay": 0.01,
    "backbone_learning_rate": 5e-06,
    "new_module_learning_rate": 0.0001,
    "adam_beta1": 0.9,
    "adam_beta2": 0.95,
    "adam_epsilon": 1e-08,
    "fused_adamw": true,
    "freeze_gate_head": true,
    "log_every_steps": 1,
    "eval_every_steps": 25,
    "checkpoint_every_steps": 250,
    "max_train_examples": null,
    "max_validation_examples": null,
    "freeze_decoder_first_fraction": 1.0,
    "probe_extra_new_tokens": 16
  },
  "phase": {
    "micro_batch_size": 16,
    "eval_batch_size": 16,
    "gradient_accumulation_steps": 4,
    "num_train_epochs": 1,
    "warmup_ratio": 0.03,
    "shuffle_train": true
  },
  "cache": {
    "preprocessed_root": "cache/preprocessed_pre_sft_multiturn"
  },
  "paths": {
    "run_root": "runs_pre_sft_multiturn",
    "export_root": "exports_multiturn"
  },
  "inference": {
    "format": "alpaca_single_turn",
    "use_base_chat_template": true
  },
  "wandb": {
    "enabled": true,
    "project": "samantha-pre-sft",
    "run_name": "t5gemma2-thoughtloop-pre-sft-everyday-all-turns"
  },
  "hub": {
    "model_repo_id": "BRlkl/test_multiturn",
    "private": false
  }
}