| { |
| "dataset": { |
| "raw_repo_id": "BRlkl/timed_natural_conversations_5min4", |
| "cleaned_repo_id": "BRlkl/timed_natural_conversations_5min4", |
| "cleaned_local_parquet_dir": "artifacts/timed_natural_conversations_5min4/parquet", |
| "source_split": "train", |
| "validation_ratio": 0.02, |
| "split_seed": 17, |
| "use_base_chat_template": true |
| }, |
| "model": { |
| "base_model_name": "google/t5gemma-l-l-prefixlm-it", |
| "initial_model_path": "BRlkl/test_multiturn2", |
| "dtype": "bfloat16", |
| "attn_implementation": "sdpa", |
| "magicnorm_eps": 1e-06, |
| "z_slots": 256, |
| "num_time_tokens": 0, |
| "use_explicit_time_features": false, |
| "gate_attention_heads": 4, |
| "max_observation_tokens": 256, |
| "max_decoder_tokens": 256, |
| "thought_loop_proposal_mode": "observation_hidden_compression", |
| "preserve_observation_encoder_manifold": true, |
| "observation_encoder_use_state_context": true, |
| "latent_attention_mask_mode": "full", |
| "initial_update_gate_bias": 6.0 |
| }, |
| "rollout": { |
| "tick_seconds": 1.0, |
| "chunk_ticks": 300, |
| "max_horizon_ticks": 300, |
| "sort_by_duration_bucket": false, |
| "drop_overlong_examples": true, |
| "long_window_strategy": "tail", |
| "post_speech_feedback_delay_seconds": 1.0 |
| }, |
| "training": { |
| "seed": 17, |
| "micro_batch_size": 12, |
| "eval_batch_size": 1, |
| "gradient_accumulation_steps": 3, |
| "num_workers": 0, |
| "num_train_epochs": 1, |
| "learning_rate": 5e-06, |
| "weight_decay": 0.05, |
| "adam_beta1": 0.9, |
| "adam_beta2": 0.95, |
| "adam_epsilon": 1e-08, |
| "warmup_ratio": 0.03, |
| "gradient_checkpointing": true, |
| "fused_adamw": true, |
| "mixed_precision": "bf16", |
| "max_grad_norm": 0.1, |
| "tbptt_start_steps": 300, |
| "tbptt_end_steps": 300, |
| "gate_loss_mode": "bce", |
| "gate_soft_target_early_window_ticks": 5, |
| "gate_soft_target_late_window_ticks": 5, |
| "gate_soft_target_tau_ticks": 1.5, |
| "gate_soft_target_shape": "gaussian", |
| "gate_soft_target_peak": 1.0, |
| "gate_soft_target_floor": 0.0, |
| "gate_soft_target_block_user_ticks": true, |
| "hazard_soft_target_window_ticks": 16, |
| "hazard_soft_target_tau_ticks": 4.0, |
| "hazard_censor_weight": 0.25, |
| "use_ranked_gate_objective": false, |
| "gate_bce_weight": 1.0, |
| "gate_ranking_weight": 1.0, |
| "gate_ranking_margin": 0.5, |
| "gate_ranking_lookback": 64, |
| "gradual_unfreeze_decoder": false, |
| "decoder_initial_trainable_fraction": 0.0, |
| "decoder_unfreeze_start_fraction": 0.0, |
| "decoder_unfreeze_end_fraction": 0.5, |
| "freeze_gate_head_first_half": false, |
| "gate_head_freeze_fraction": 0.5, |
| "gate_teacher_logit_scale": 20.0, |
| "gate_positive_weight": 16.0, |
| "use_dynamic_gate_positive_weight": false, |
| "dynamic_gate_positive_weight_min": 8.0, |
| "dynamic_gate_positive_weight_max": 16.0, |
| "gate_loss_weight": 0.75, |
| "decoder_loss_weight": 1.0, |
| "state_delta_weight": 0.1, |
| "shuffle_train": true, |
| "sort_dataset_by_length": false, |
| "logging_steps": 1, |
| "eval_steps": 51, |
| "checkpoint_steps": 50, |
| "eval_max_batches": 12 |
| }, |
| "cache": { |
| "preprocessed_root": "cache/preprocessed_5min" |
| }, |
| "paths": { |
| "run_root": "runs", |
| "export_root": "exports" |
| }, |
| "inference": { |
| "use_base_chat_template": true |
| }, |
| "wandb": { |
| "enabled": true, |
| "project": "samantha-sft-5min", |
| "run_name": "t5gemma2-thoughtloop-5min-from-multiturn3" |
| }, |
| "hub": { |
| "model_repo_id": "BRlkl/test-true2", |
| "private": false, |
| "push_checkpoints": false, |
| "checkpoint_path_prefix": "checkpoints" |
| } |
| } |
