runs/20260813T154138Z/arms/control/adapter/debug.log

[2026-08-13 15:48:49,616] [DEBUG] [axolotl.utils.config.log_gpu_memory_usage:127] [PID:1747] baseline 0.000GB ()
[2026-08-13 15:48:49,617] [INFO] [axolotl.cli.config.load_cfg:333] [PID:1747] config:
{
  "activation_offloading": false,
  "adapter": "lora",
  "attn_implementation": "flash_attention_2",
  "attn_needs_dtype_cast": true,
  "attn_supports_packing": true,
  "attn_uses_flash_lib": true,
  "axolotl_config_path": "/workspace/python4-aft-v2-20260813T154138Z-control/experiments/python4/aft_v2/runs/20260813T154138Z/arms/control/train/axolotl.yaml",
  "base_model": "/workspace/python4-aft-v2-state/20260813T154138Z/control/parent/control/sft/end",
  "base_model_config": "/workspace/python4-aft-v2-state/20260813T154138Z/control/parent/control/sft/end",
  "batch_size": 32,
  "bf16": true,
  "capabilities": {
    "bf16": true,
    "compute_capability": "sm_90",
    "fp8": true,
    "n_gpu": 1,
    "n_node": 1,
    "tf32": true
  },
  "chat_template": "gemma3",
  "checkpoint_schedule": [
    128
  ],
  "context_parallel_size": 1,
  "cosine_min_lr_ratio": 0.1,
  "dataloader_num_workers": 1,
  "dataloader_pin_memory": true,
  "dataloader_prefetch_factor": 256,
  "dataset_num_proc": 16,
  "dataset_prepared_path": "/workspace/python4-aft-v2-20260813T154138Z-control/experiments/python4/aft_v2/runs/20260813T154138Z/arms/control/train/prepared",
  "datasets": [
    {
      "chat_template": "tokenizer_default",
      "field_messages": "messages",
      "message_property_mappings": {
        "content": "content",
        "role": "role"
      },
      "path": "/workspace/python4-aft-v2-20260813T154138Z-control/experiments/python4/aft_v2/runs/20260813T154138Z/arms/control/aft_training.jsonl",
      "trust_remote_code": false,
      "type": "chat_template"
    }
  ],
  "ddp": false,
  "ddp_find_unused_parameters": true,
  "device": "cuda:0",
  "dion_rank_fraction": 1.0,
  "dion_rank_multiple_of": 1,
  "eaft_alpha": 1.0,
  "eaft_k": 20,
  "env_capabilities": {
    "torch_version": "2.12.1"
  },
  "eot_tokens": [
    "<end_of_turn>"
  ],
  "eval_batch_size": 4,
  "eval_causal_lm_metrics": [
    "sacrebleu",
    "comet",
    "ter",
    "chrf"
  ],
  "eval_max_new_tokens": 128,
  "eval_table_size": 0,
  "experimental_skip_move_to_device": true,
  "fp16": false,
  "generate_samples": false,
  "generation_do_sample": true,
  "generation_max_new_tokens": 50,
  "generation_prompt_ratio": 0.5,
  "generation_temperature": 0.7,
  "gradient_accumulation_steps": 8,
  "gradient_checkpointing": true,
  "gradient_checkpointing_kwargs": {
    "use_reentrant": false
  },
  "include_tkps": true,
  "is_multimodal": true,
  "layer_offloading": false,
  "learning_rate": 0.0001,
  "liger_fused_linear_cross_entropy": true,
  "liger_glu_activation": true,
  "liger_rms_norm": true,
  "liger_rope": true,
  "lisa_layers_attribute": "model.layers",
  "load_best_model_at_end": false,
  "load_in_4bit": false,
  "load_in_8bit": false,
  "local_rank": 0,
  "logging_steps": 1,
  "lora_alpha": 128,
  "lora_dropout": 0.0,
  "lora_embedding_kernel": true,
  "lora_mlp_kernel": true,
  "lora_o_kernel": true,
  "lora_qkv_kernel": true,
  "lora_r": 64,
  "lora_target_modules": [
    "model.language_model.layers.0.self_attn.q_proj",
    "model.language_model.layers.0.self_attn.k_proj",
    "model.language_model.layers.0.self_attn.v_proj",
    "model.language_model.layers.0.self_attn.o_proj",
    "model.language_model.layers.0.mlp.gate_proj",
    "model.language_model.layers.0.mlp.up_proj",
    "model.language_model.layers.0.mlp.down_proj",
    "model.language_model.layers.1.self_attn.q_proj",
    "model.language_model.layers.1.self_attn.k_proj",
    "model.language_model.layers.1.self_attn.v_proj",
    "model.language_model.layers.1.self_attn.o_proj",
    "model.language_model.layers.1.mlp.gate_proj",
    "model.language_model.layers.1.mlp.up_proj",
    "model.language_model.layers.1.mlp.down_proj",
    "model.language_model.layers.2.self_attn.q_proj",
    "model.language_model.layers.2.self_attn.k_proj",
    "model.language_model.layers.2.self_attn.v_proj",
    "model.language_model.layers.2.self_attn.o_proj",
    "model.language_model.layers.2.mlp.gate_proj",
    "model.language_model.layers.2.mlp.up_proj",
    "model.language_model.layers.2.mlp.down_proj",
    "model.language_model.layers.3.self_attn.q_proj",
    "model.language_model.layers.3.self_attn.k_proj",
    "model.language_model.layers.3.self_attn.v_proj",
    "model.language_model.layers.3.self_attn.o_proj",
    "model.language_model.layers.3.mlp.gate_proj",
    "model.language_model.layers.3.mlp.up_proj",
    "model.language_model.layers.3.mlp.down_proj",
    "model.language_model.layers.4.self_attn.q_proj",
    "model.language_model.layers.4.self_attn.k_proj",
    "model.language_model.layers.4.self_attn.v_proj",
    "model.language_model.layers.4.self_attn.o_proj",
    "model.language_model.layers.4.mlp.gate_proj",
    "model.language_model.layers.4.mlp.up_proj",
    "model.language_model.layers.4.mlp.down_proj",
    "model.language_model.layers.5.self_attn.q_proj",
    "model.language_model.layers.5.self_attn.k_proj",
    "model.language_model.layers.5.self_attn.v_proj",
    "model.language_model.layers.5.self_attn.o_proj",
    "model.language_model.layers.5.mlp.gate_proj",
    "model.language_model.layers.5.mlp.up_proj",
    "model.language_model.layers.5.mlp.down_proj",
    "model.language_model.layers.6.self_attn.q_proj",
    "model.language_model.layers.6.self_attn.k_proj",
    "model.language_model.layers.6.self_attn.v_proj",
    "model.language_model.layers.6.self_attn.o_proj",
    "model.language_model.layers.6.mlp.gate_proj",
    "model.language_model.layers.6.mlp.up_proj",
    "model.language_model.layers.6.mlp.down_proj",
    "model.language_model.layers.7.self_attn.q_proj",
    "model.language_model.layers.7.self_attn.k_proj",
    "model.language_model.layers.7.self_attn.v_proj",
    "model.language_model.layers.7.self_attn.o_proj",
    "model.language_model.layers.7.mlp.gate_proj",
    "model.language_model.layers.7.mlp.up_proj",
    "model.language_model.layers.7.mlp.down_proj",
    "model.language_model.layers.8.self_attn.q_proj",
    "model.language_model.layers.8.self_attn.k_proj",
    "model.language_model.layers.8.self_attn.v_proj",
    "model.language_model.layers.8.self_attn.o_proj",
    "model.language_model.layers.8.mlp.gate_proj",
    "model.language_model.layers.8.mlp.up_proj",
    "model.language_model.layers.8.mlp.down_proj",
    "model.language_model.layers.9.self_attn.q_proj",
    "model.language_model.layers.9.self_attn.k_proj",
    "model.language_model.layers.9.self_attn.v_proj",
    "model.language_model.layers.9.self_attn.o_proj",
    "model.language_model.layers.9.mlp.gate_proj",
    "model.language_model.layers.9.mlp.up_proj",
    "model.language_model.layers.9.mlp.down_proj",
    "model.language_model.layers.10.self_attn.q_proj",
    "model.language_model.layers.10.self_attn.k_proj",
    "model.language_model.layers.10.self_attn.v_proj",
    "model.language_model.layers.10.self_attn.o_proj",
    "model.language_model.layers.10.mlp.gate_proj",
    "model.language_model.layers.10.mlp.up_proj",
    "model.language_model.layers.10.mlp.down_proj",
    "model.language_model.layers.11.self_attn.q_proj",
    "model.language_model.layers.11.self_attn.k_proj",
    "model.language_model.layers.11.self_attn.v_proj",
    "model.language_model.layers.11.self_attn.o_proj",
    "model.language_model.layers.11.mlp.gate_proj",
    "model.language_model.layers.11.mlp.up_proj",
    "model.language_model.layers.11.mlp.down_proj",
    "model.language_model.layers.12.self_attn.q_proj",
    "model.language_model.layers.12.self_attn.k_proj",
    "model.language_model.layers.12.self_attn.v_proj",
    "model.language_model.layers.12.self_attn.o_proj",
    "model.language_model.layers.12.mlp.gate_proj",
    "model.language_model.layers.12.mlp.up_proj",
    "model.language_model.layers.12.mlp.down_proj",
    "model.language_model.layers.13.self_attn.q_proj",
    "model.language_model.layers.13.self_attn.k_proj",
    "model.language_model.layers.13.self_attn.v_proj",
    "model.language_model.layers.13.self_attn.o_proj",
    "model.language_model.layers.13.mlp.gate_proj",
    "model.language_model.layers.13.mlp.up_proj",
    "model.language_model.layers.13.mlp.down_proj",
    "model.language_model.layers.14.self_attn.q_proj",
    "model.language_model.layers.14.self_attn.k_proj",
    "model.language_model.layers.14.self_attn.v_proj",
    "model.language_model.layers.14.self_attn.o_proj",
    "model.language_model.layers.14.mlp.gate_proj",
    "model.language_model.layers.14.mlp.up_proj",
    "model.language_model.layers.14.mlp.down_proj",
    "model.language_model.layers.15.self_attn.q_proj",
    "model.language_model.layers.15.self_attn.k_proj",
    "model.language_model.layers.15.self_attn.v_proj",
    "model.language_model.layers.15.self_attn.o_proj",
    "model.language_model.layers.15.mlp.gate_proj",
    "model.language_model.layers.15.mlp.up_proj",
    "model.language_model.layers.15.mlp.down_proj",
    "model.language_model.layers.16.self_attn.q_proj",
    "model.language_model.layers.16.self_attn.k_proj",
    "model.language_model.layers.16.self_attn.v_proj",
    "model.language_model.layers.16.self_attn.o_proj",
    "model.language_model.layers.16.mlp.gate_proj",
    "model.language_model.layers.16.mlp.up_proj",
    "model.language_model.layers.16.mlp.down_proj",
    "model.language_model.layers.17.self_attn.q_proj",
    "model.language_model.layers.17.self_attn.k_proj",
    "model.language_model.layers.17.self_attn.v_proj",
    "model.language_model.layers.17.self_attn.o_proj",
    "model.language_model.layers.17.mlp.gate_proj",
    "model.language_model.layers.17.mlp.up_proj",
    "model.language_model.layers.17.mlp.down_proj",
    "model.language_model.layers.18.self_attn.q_proj",
    "model.language_model.layers.18.self_attn.k_proj",
    "model.language_model.layers.18.self_attn.v_proj",
    "model.language_model.layers.18.self_attn.o_proj",
    "model.language_model.layers.18.mlp.gate_proj",
    "model.language_model.layers.18.mlp.up_proj",
    "model.language_model.layers.18.mlp.down_proj",
    "model.language_model.layers.19.self_attn.q_proj",
    "model.language_model.layers.19.self_attn.k_proj",
    "model.language_model.layers.19.self_attn.v_proj",
    "model.language_model.layers.19.self_attn.o_proj",
    "model.language_model.layers.19.mlp.gate_proj",
    "model.language_model.layers.19.mlp.up_proj",
    "model.language_model.layers.19.mlp.down_proj",
    "model.language_model.layers.20.self_attn.q_proj",
    "model.language_model.layers.20.self_attn.k_proj",
    "model.language_model.layers.20.self_attn.v_proj",
    "model.language_model.layers.20.self_attn.o_proj",
    "model.language_model.layers.20.mlp.gate_proj",
    "model.language_model.layers.20.mlp.up_proj",
    "model.language_model.layers.20.mlp.down_proj",
    "model.language_model.layers.21.self_attn.q_proj",
    "model.language_model.layers.21.self_attn.k_proj",
    "model.language_model.layers.21.self_attn.v_proj",
    "model.language_model.layers.21.self_attn.o_proj",
    "model.language_model.layers.21.mlp.gate_proj",
    "model.language_model.layers.21.mlp.up_proj",
    "model.language_model.layers.21.mlp.down_proj",
    "model.language_model.layers.22.self_attn.q_proj",
    "model.language_model.layers.22.self_attn.k_proj",
    "model.language_model.layers.22.self_attn.v_proj",
    "model.language_model.layers.22.self_attn.o_proj",
    "model.language_model.layers.22.mlp.gate_proj",
    "model.language_model.layers.22.mlp.up_proj",
    "model.language_model.layers.22.mlp.down_proj",
    "model.language_model.layers.23.self_attn.q_proj",
    "model.language_model.layers.23.self_attn.k_proj",
    "model.language_model.layers.23.self_attn.v_proj",
    "model.language_model.layers.23.self_attn.o_proj",
    "model.language_model.layers.23.mlp.gate_proj",
    "model.language_model.layers.23.mlp.up_proj",
    "model.language_model.layers.23.mlp.down_proj",
    "model.language_model.layers.24.self_attn.q_proj",
    "model.language_model.layers.24.self_attn.k_proj",
    "model.language_model.layers.24.self_attn.v_proj",
    "model.language_model.layers.24.self_attn.o_proj",
    "model.language_model.layers.24.mlp.gate_proj",
    "model.language_model.layers.24.mlp.up_proj",
    "model.language_model.layers.24.mlp.down_proj",
    "model.language_model.layers.25.self_attn.q_proj",
    "model.language_model.layers.25.self_attn.k_proj",
    "model.language_model.layers.25.self_attn.v_proj",
    "model.language_model.layers.25.self_attn.o_proj",
    "model.language_model.layers.25.mlp.gate_proj",
    "model.language_model.layers.25.mlp.up_proj",
    "model.language_model.layers.25.mlp.down_proj",
    "model.language_model.layers.26.self_attn.q_proj",
    "model.language_model.layers.26.self_attn.k_proj",
    "model.language_model.layers.26.self_attn.v_proj",
    "model.language_model.layers.26.self_attn.o_proj",
    "model.language_model.layers.26.mlp.gate_proj",
    "model.language_model.layers.26.mlp.up_proj",
    "model.language_model.layers.26.mlp.down_proj",
    "model.language_model.layers.27.self_attn.q_proj",
    "model.language_model.layers.27.self_attn.k_proj",
    "model.language_model.layers.27.self_attn.v_proj",
    "model.language_model.layers.27.self_attn.o_proj",
    "model.language_model.layers.27.mlp.gate_proj",
    "model.language_model.layers.27.mlp.up_proj",
    "model.language_model.layers.27.mlp.down_proj",
    "model.language_model.layers.28.self_attn.q_proj",
    "model.language_model.layers.28.self_attn.k_proj",
    "model.language_model.layers.28.self_attn.v_proj",
    "model.language_model.layers.28.self_attn.o_proj",
    "model.language_model.layers.28.mlp.gate_proj",
    "model.language_model.layers.28.mlp.up_proj",
    "model.language_model.layers.28.mlp.down_proj",
    "model.language_model.layers.29.self_attn.q_proj",
    "model.language_model.layers.29.self_attn.k_proj",
    "model.language_model.layers.29.self_attn.v_proj",
    "model.language_model.layers.29.self_attn.o_proj",
    "model.language_model.layers.29.mlp.gate_proj",
    "model.language_model.layers.29.mlp.up_proj",
    "model.language_model.layers.29.mlp.down_proj",
    "model.language_model.layers.30.self_attn.q_proj",
    "model.language_model.layers.30.self_attn.k_proj",
    "model.language_model.layers.30.self_attn.v_proj",
    "model.language_model.layers.30.self_attn.o_proj",
    "model.language_model.layers.30.mlp.gate_proj",
    "model.language_model.layers.30.mlp.up_proj",
    "model.language_model.layers.30.mlp.down_proj",
    "model.language_model.layers.31.self_attn.q_proj",
    "model.language_model.layers.31.self_attn.k_proj",
    "model.language_model.layers.31.self_attn.v_proj",
    "model.language_model.layers.31.self_attn.o_proj",
    "model.language_model.layers.31.mlp.gate_proj",
    "model.language_model.layers.31.mlp.up_proj",
    "model.language_model.layers.31.mlp.down_proj",
    "model.language_model.layers.32.self_attn.q_proj",
    "model.language_model.layers.32.self_attn.k_proj",
    "model.language_model.layers.32.self_attn.v_proj",
    "model.language_model.layers.32.self_attn.o_proj",
    "model.language_model.layers.32.mlp.gate_proj",
    "model.language_model.layers.32.mlp.up_proj",
    "model.language_model.layers.32.mlp.down_proj",
    "model.language_model.layers.33.self_attn.q_proj",
    "model.language_model.layers.33.self_attn.k_proj",
    "model.language_model.layers.33.self_attn.v_proj",
    "model.language_model.layers.33.self_attn.o_proj",
    "model.language_model.layers.33.mlp.gate_proj",
    "model.language_model.layers.33.mlp.up_proj",
    "model.language_model.layers.33.mlp.down_proj",
    "model.language_model.layers.34.self_attn.q_proj",
    "model.language_model.layers.34.self_attn.k_proj",
    "model.language_model.layers.34.self_attn.v_proj",
    "model.language_model.layers.34.self_attn.o_proj",
    "model.language_model.layers.34.mlp.gate_proj",
    "model.language_model.layers.34.mlp.up_proj",
    "model.language_model.layers.34.mlp.down_proj",
    "model.language_model.layers.35.self_attn.q_proj",
    "model.language_model.layers.35.self_attn.k_proj",
    "model.language_model.layers.35.self_attn.v_proj",
    "model.language_model.layers.35.self_attn.o_proj",
    "model.language_model.layers.35.mlp.gate_proj",
    "model.language_model.layers.35.mlp.up_proj",
    "model.language_model.layers.35.mlp.down_proj",
    "model.language_model.layers.36.self_attn.q_proj",
    "model.language_model.layers.36.self_attn.k_proj",
    "model.language_model.layers.36.self_attn.v_proj",
    "model.language_model.layers.36.self_attn.o_proj",
    "model.language_model.layers.36.mlp.gate_proj",
    "model.language_model.layers.36.mlp.up_proj",
    "model.language_model.layers.36.mlp.down_proj",
    "model.language_model.layers.37.self_attn.q_proj",
    "model.language_model.layers.37.self_attn.k_proj",
    "model.language_model.layers.37.self_attn.v_proj",
    "model.language_model.layers.37.self_attn.o_proj",
    "model.language_model.layers.37.mlp.gate_proj",
    "model.language_model.layers.37.mlp.up_proj",
    "model.language_model.layers.37.mlp.down_proj",
    "model.language_model.layers.38.self_attn.q_proj",
    "model.language_model.layers.38.self_attn.k_proj",
    "model.language_model.layers.38.self_attn.v_proj",
    "model.language_model.layers.38.self_attn.o_proj",
    "model.language_model.layers.38.mlp.gate_proj",
    "model.language_model.layers.38.mlp.up_proj",
    "model.language_model.layers.38.mlp.down_proj",
    "model.language_model.layers.39.self_attn.q_proj",
    "model.language_model.layers.39.self_attn.k_proj",
    "model.language_model.layers.39.self_attn.v_proj",
    "model.language_model.layers.39.self_attn.o_proj",
    "model.language_model.layers.39.mlp.gate_proj",
    "model.language_model.layers.39.mlp.up_proj",
    "model.language_model.layers.39.mlp.down_proj",
    "model.language_model.layers.40.self_attn.q_proj",
    "model.language_model.layers.40.self_attn.k_proj",
    "model.language_model.layers.40.self_attn.v_proj",
    "model.language_model.layers.40.self_attn.o_proj",
    "model.language_model.layers.40.mlp.gate_proj",
    "model.language_model.layers.40.mlp.up_proj",
    "model.language_model.layers.40.mlp.down_proj",
    "model.language_model.layers.41.self_attn.q_proj",
    "model.language_model.layers.41.self_attn.k_proj",
    "model.language_model.layers.41.self_attn.v_proj",
    "model.language_model.layers.41.self_attn.o_proj",
    "model.language_model.layers.41.mlp.gate_proj",
    "model.language_model.layers.41.mlp.up_proj",
    "model.language_model.layers.41.mlp.down_proj",
    "model.language_model.layers.42.self_attn.q_proj",
    "model.language_model.layers.42.self_attn.k_proj",
    "model.language_model.layers.42.self_attn.v_proj",
    "model.language_model.layers.42.self_attn.o_proj",
    "model.language_model.layers.42.mlp.gate_proj",
    "model.language_model.layers.42.mlp.up_proj",
    "model.language_model.layers.42.mlp.down_proj",
    "model.language_model.layers.43.self_attn.q_proj",
    "model.language_model.layers.43.self_attn.k_proj",
    "model.language_model.layers.43.self_attn.v_proj",
    "model.language_model.layers.43.self_attn.o_proj",
    "model.language_model.layers.43.mlp.gate_proj",
    "model.language_model.layers.43.mlp.up_proj",
    "model.language_model.layers.43.mlp.down_proj",
    "model.language_model.layers.44.self_attn.q_proj",
    "model.language_model.layers.44.self_attn.k_proj",
    "model.language_model.layers.44.self_attn.v_proj",
    "model.language_model.layers.44.self_attn.o_proj",
    "model.language_model.layers.44.mlp.gate_proj",
    "model.language_model.layers.44.mlp.up_proj",
    "model.language_model.layers.44.mlp.down_proj",
    "model.language_model.layers.45.self_attn.q_proj",
    "model.language_model.layers.45.self_attn.k_proj",
    "model.language_model.layers.45.self_attn.v_proj",
    "model.language_model.layers.45.self_attn.o_proj",
    "model.language_model.layers.45.mlp.gate_proj",
    "model.language_model.layers.45.mlp.up_proj",
    "model.language_model.layers.45.mlp.down_proj",
    "model.language_model.layers.46.self_attn.q_proj",
    "model.language_model.layers.46.self_attn.k_proj",
    "model.language_model.layers.46.self_attn.v_proj",
    "model.language_model.layers.46.self_attn.o_proj",
    "model.language_model.layers.46.mlp.gate_proj",
    "model.language_model.layers.46.mlp.up_proj",
    "model.language_model.layers.46.mlp.down_proj",
    "model.language_model.layers.47.self_attn.q_proj",
    "model.language_model.layers.47.self_attn.k_proj",
    "model.language_model.layers.47.self_attn.v_proj",
    "model.language_model.layers.47.self_attn.o_proj",
    "model.language_model.layers.47.mlp.gate_proj",
    "model.language_model.layers.47.mlp.up_proj",
    "model.language_model.layers.47.mlp.down_proj",
    "model.language_model.layers.48.self_attn.q_proj",
    "model.language_model.layers.48.self_attn.k_proj",
    "model.language_model.layers.48.self_attn.v_proj",
    "model.language_model.layers.48.self_attn.o_proj",
    "model.language_model.layers.48.mlp.gate_proj",
    "model.language_model.layers.48.mlp.up_proj",
    "model.language_model.layers.48.mlp.down_proj",
    "model.language_model.layers.49.self_attn.q_proj",
    "model.language_model.layers.49.self_attn.k_proj",
    "model.language_model.layers.49.self_attn.v_proj",
    "model.language_model.layers.49.self_attn.o_proj",
    "model.language_model.layers.49.mlp.gate_proj",
    "model.language_model.layers.49.mlp.up_proj",
    "model.language_model.layers.49.mlp.down_proj",
    "model.language_model.layers.50.self_attn.q_proj",
    "model.language_model.layers.50.self_attn.k_proj",
    "model.language_model.layers.50.self_attn.v_proj",
    "model.language_model.layers.50.self_attn.o_proj",
    "model.language_model.layers.50.mlp.gate_proj",
    "model.language_model.layers.50.mlp.up_proj",
    "model.language_model.layers.50.mlp.down_proj",
    "model.language_model.layers.51.self_attn.q_proj",
    "model.language_model.layers.51.self_attn.k_proj",
    "model.language_model.layers.51.self_attn.v_proj",
    "model.language_model.layers.51.self_attn.o_proj",
    "model.language_model.layers.51.mlp.gate_proj",
    "model.language_model.layers.51.mlp.up_proj",
    "model.language_model.layers.51.mlp.down_proj",
    "model.language_model.layers.52.self_attn.q_proj",
    "model.language_model.layers.52.self_attn.k_proj",
    "model.language_model.layers.52.self_attn.v_proj",
    "model.language_model.layers.52.self_attn.o_proj",
    "model.language_model.layers.52.mlp.gate_proj",
    "model.language_model.layers.52.mlp.up_proj",
    "model.language_model.layers.52.mlp.down_proj",
    "model.language_model.layers.53.self_attn.q_proj",
    "model.language_model.layers.53.self_attn.k_proj",
    "model.language_model.layers.53.self_attn.v_proj",
    "model.language_model.layers.53.self_attn.o_proj",
    "model.language_model.layers.53.mlp.gate_proj",
    "model.language_model.layers.53.mlp.up_proj",
    "model.language_model.layers.53.mlp.down_proj",
    "model.language_model.layers.54.self_attn.q_proj",
    "model.language_model.layers.54.self_attn.k_proj",
    "model.language_model.layers.54.self_attn.v_proj",
    "model.language_model.layers.54.self_attn.o_proj",
    "model.language_model.layers.54.mlp.gate_proj",
    "model.language_model.layers.54.mlp.up_proj",
    "model.language_model.layers.54.mlp.down_proj",
    "model.language_model.layers.55.self_attn.q_proj",
    "model.language_model.layers.55.self_attn.k_proj",
    "model.language_model.layers.55.self_attn.v_proj",
    "model.language_model.layers.55.self_attn.o_proj",
    "model.language_model.layers.55.mlp.gate_proj",
    "model.language_model.layers.55.mlp.up_proj",
    "model.language_model.layers.55.mlp.down_proj",
    "model.language_model.layers.56.self_attn.q_proj",
    "model.language_model.layers.56.self_attn.k_proj",
    "model.language_model.layers.56.self_attn.v_proj",
    "model.language_model.layers.56.self_attn.o_proj",
    "model.language_model.layers.56.mlp.gate_proj",
    "model.language_model.layers.56.mlp.up_proj",
    "model.language_model.layers.56.mlp.down_proj",
    "model.language_model.layers.57.self_attn.q_proj",
    "model.language_model.layers.57.self_attn.k_proj",
    "model.language_model.layers.57.self_attn.v_proj",
    "model.language_model.layers.57.self_attn.o_proj",
    "model.language_model.layers.57.mlp.gate_proj",
    "model.language_model.layers.57.mlp.up_proj",
    "model.language_model.layers.57.mlp.down_proj",
    "model.language_model.layers.58.self_attn.q_proj",
    "model.language_model.layers.58.self_attn.k_proj",
    "model.language_model.layers.58.self_attn.v_proj",
    "model.language_model.layers.58.self_attn.o_proj",
    "model.language_model.layers.58.mlp.gate_proj",
    "model.language_model.layers.58.mlp.up_proj",
    "model.language_model.layers.58.mlp.down_proj",
    "model.language_model.layers.59.self_attn.q_proj",
    "model.language_model.layers.59.self_attn.k_proj",
    "model.language_model.layers.59.self_attn.v_proj",
    "model.language_model.layers.59.self_attn.o_proj",
    "model.language_model.layers.59.mlp.gate_proj",
    "model.language_model.layers.59.mlp.up_proj",
    "model.language_model.layers.59.mlp.down_proj",
    "model.language_model.layers.60.self_attn.q_proj",
    "model.language_model.layers.60.self_attn.k_proj",
    "model.language_model.layers.60.self_attn.v_proj",
    "model.language_model.layers.60.self_attn.o_proj",
    "model.language_model.layers.60.mlp.gate_proj",
    "model.language_model.layers.60.mlp.up_proj",
    "model.language_model.layers.60.mlp.down_proj",
    "model.language_model.layers.61.self_attn.q_proj",
    "model.language_model.layers.61.self_attn.k_proj",
    "model.language_model.layers.61.self_attn.v_proj",
    "model.language_model.layers.61.self_attn.o_proj",
    "model.language_model.layers.61.mlp.gate_proj",
    "model.language_model.layers.61.mlp.up_proj",
    "model.language_model.layers.61.mlp.down_proj"
  ],
  "loraplus_lr_embedding": 1e-06,
  "lr_scheduler": "cosine",
  "max_grad_norm": 1.0,
  "mean_resizing_embeddings": false,
  "merge_method": "memory_efficient",
  "micro_batch_size": 4,
  "model_config_type": "gemma3",
  "model_config_type_text": "gemma3_text",
  "num_epochs": 4.0,
  "num_generation_samples": 3,
  "optimizer": "adamw_torch_fused",
  "otel_metrics_host": "localhost",
  "otel_metrics_port": 8000,
  "output_dir": "/workspace/python4-aft-v2-20260813T154138Z-control/experiments/python4/aft_v2/runs/20260813T154138Z/arms/control/train/checkpoints",
  "pad_to_sequence_len": false,
  "plugins": [
    "axolotl.integrations.liger.LigerPlugin",
    "scimt.train.axolotl_plugins.CheckpointSchedulePlugin"
  ],
  "pretrain_multipack_attn": true,
  "processor_config": "/workspace/python4-aft-v2-state/20260813T154138Z/control/parent/control/sft/end",
  "profiler_steps_start": 0,
  "qgalore_cos_threshold": 0.4,
  "qgalore_gamma_proj": 2,
  "qgalore_proj_bits": 4,
  "qgalore_proj_group_size": 256,
  "qgalore_proj_quant": true,
  "qgalore_proj_type": "std",
  "qgalore_queue_size": 5,
  "qgalore_rank": 256,
  "qgalore_scale": 0.25,
  "qgalore_update_proj_gap": 200,
  "qlora_sharded_model_loading": false,
  "quantize_moe_experts": false,
  "ray_num_workers": 1,
  "relora_prune_method": "magnitude",
  "resources_per_worker": {
    "GPU": 1
  },
  "sample_packing": false,
  "sample_packing_bin_size": 200,
  "sample_packing_group_size": 100000,
  "save_only_model": true,
  "save_safetensors": true,
  "save_strategy": "no",
  "save_total_limit": 1,
  "seed": 424242,
  "sequence_len": 4096,
  "shuffle_before_merging_datasets": false,
  "shuffle_merged_datasets": true,
  "skip_prepare_dataset": false,
  "streaming_multipack_buffer_size": 10000,
  "strict": false,
  "tensor_parallel_size": 1,
  "tf32": true,
  "tiled_mlp_use_original_mlp": true,
  "tokenizer_config": "/workspace/python4-aft-v2-state/20260813T154138Z/control/parent/control/sft/end",
  "tokenizer_save_jinja_files": true,
  "torch_dtype": "torch.bfloat16",
  "train_on_inputs": false,
  "trl": {
    "async_prefetch": false,
    "log_completions": false,
    "mask_truncated_completions": false,
    "ref_model_mixup_alpha": 0.9,
    "ref_model_sync_steps": 64,
    "replay_buffer_size": 0,
    "replay_recompute_logps": true,
    "reroll_max_groups": 1,
    "reroll_start_fraction": 1.0,
    "reward_num_workers": 1,
    "scale_rewards": true,
    "skip_zero_advantage_batches": true,
    "sync_ref_model": false,
    "use_data_producer": false,
    "use_vllm": false,
    "vllm_lora_sync": false,
    "vllm_server_host": "0.0.0.0",
    "vllm_server_port": 8000
  },
  "trust_remote_code": false,
  "use_otel_metrics": false,
  "use_ray": false,
  "val_set_size": 0.0,
  "vllm": {
    "device": "auto",
    "dtype": "auto",
    "gpu_memory_utilization": 0.9,
    "host": "0.0.0.0",
    "port": 8000
  },
  "warmup_ratio": 0.05,
  "weight_decay": 0.01,
  "world_size": 1
}
[2026-08-13 15:48:50,617] [DEBUG] [axolotl.loaders.utils.check_model_config:88] [PID:1747] Loaded image size: 896 from model config
[2026-08-13 15:48:52,258] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:311] [PID:1747] EOS: 106 / <end_of_turn>
[2026-08-13 15:48:52,258] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:312] [PID:1747] BOS: 2 / <bos>
[2026-08-13 15:48:52,258] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:313] [PID:1747] PAD: 0 / <pad>
[2026-08-13 15:48:52,258] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:314] [PID:1747] UNK: 3 / <unk>
[2026-08-13 15:48:52,259] [INFO] [axolotl.utils.data.shared.load_preprocessed_dataset:482] [PID:1747] Unable to find prepared dataset in /workspace/python4-aft-v2-20260813T154138Z-control/experiments/python4/aft_v2/runs/20260813T154138Z/arms/control/train/prepared/cadb634638434a88a5119b219d5e7b61
[2026-08-13 15:48:52,259] [INFO] [axolotl.utils.data.sft._load_raw_datasets:320] [PID:1747] Loading raw datasets...
[2026-08-13 15:48:52,259] [WARNING] [axolotl.utils.data.sft._load_raw_datasets:322] [PID:1747] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`.

Generating train split: 0 examples [00:00, ? examples/s]
Generating train split: 1024 examples [00:00, 171750.60 examples/s]
[2026-08-13 15:48:52,729] [INFO] [axolotl.utils.data.wrappers.get_dataset_wrapper:87] [PID:1747] Loading dataset: /workspace/python4-aft-v2-20260813T154138Z-control/experiments/python4/aft_v2/runs/20260813T154138Z/arms/control/aft_training.jsonl with base_type: chat_template and prompt_style: None
[2026-08-13 15:48:52,736] [INFO] [axolotl.prompt_strategies.chat_template.__call__:1209] [PID:1747] Using chat template:
---
{{ bos_token }}
{%- if messages[0]['role'] == 'system' -%}
    {%- if messages[0]['content'] is string -%}
        {%- set first_user_prefix = messages[0]['content'] + '

' -%}
    {%- else -%}
        {%- set first_user_prefix = messages[0]['content'][0]['text'] + '

' -%}
    {%- endif -%}
    {%- set loop_messages = messages[1:] -%}
{%- else -%}
    {%- set first_user_prefix = "" -%}
    {%- set loop_messages = messages -%}
{%- endif -%}
{%- for message in loop_messages -%}
    {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%}
        {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }}
    {%- endif -%}
    {%- if (message['role'] == 'assistant') -%}
        {%- set role = "model" -%}
    {%- else -%}
        {%- set role = message['role'] -%}
    {%- endif -%}
    {{ '<start_of_turn>' + role + '
' + (first_user_prefix if loop.first else "") }}
    {%- if message['content'] is string -%}
        {{ message['content'] | trim }}
    {%- elif message['content'] is iterable -%}
        {%- for item in message['content'] -%}
            {%- if item['type'] == 'image' -%}
                {{ '<start_of_image>' }}
            {%- elif item['type'] == 'text' -%}
                {{ item['text'] | trim }}
            {%- endif -%}
        {%- endfor -%}
    {%- else -%}
        {{ raise_exception("Invalid content type") }}
    {%- endif -%}
    {{ '<end_of_turn>
' }}
{%- endfor -%}
{%- if add_generation_prompt -%}
    {{'<start_of_turn>model
'}}
{%- endif -%}

---

Tokenizing Prompts (num_proc=16):   0%|          | 0/1024 [00:00<?, ? examples/s]
Tokenizing Prompts (num_proc=16):   6%|▋         | 64/1024 [00:05<01:28, 10.89 examples/s]
Tokenizing Prompts (num_proc=16):  12%|█▎        | 128/1024 [00:09<01:04, 13.82 examples/s]
Tokenizing Prompts (num_proc=16):  19%|█▉        | 192/1024 [00:13<00:56, 14.84 examples/s]
Tokenizing Prompts (num_proc=16):  25%|██▌       | 256/1024 [00:17<00:51, 15.06 examples/s]
Tokenizing Prompts (num_proc=16):  31%|███▏      | 320/1024 [00:21<00:45, 15.47 examples/s]
Tokenizing Prompts (num_proc=16):  38%|███▊      | 384/1024 [00:25<00:40, 15.84 examples/s]
Tokenizing Prompts (num_proc=16):  44%|████▍     | 448/1024 [00:29<00:35, 16.04 examples/s]
Tokenizing Prompts (num_proc=16):  50%|█████     | 512/1024 [00:33<00:31, 16.17 examples/s]
Tokenizing Prompts (num_proc=16):  56%|█████▋    | 576/1024 [00:37<00:27, 16.14 examples/s]
Tokenizing Prompts (num_proc=16):  62%|██████▎   | 640/1024 [00:41<00:23, 16.19 examples/s]
Tokenizing Prompts (num_proc=16):  69%|██████▉   | 704/1024 [00:45<00:19, 16.20 examples/s]
Tokenizing Prompts (num_proc=16):  75%|███████▌  | 768/1024 [00:48<00:15, 16.37 examples/s]
Tokenizing Prompts (num_proc=16):  81%|████████▏ | 832/1024 [00:52<00:11, 16.28 examples/s]
Tokenizing Prompts (num_proc=16):  88%|████████▊ | 896/1024 [00:56<00:07, 16.37 examples/s]
Tokenizing Prompts (num_proc=16):  94%|█████████▍| 960/1024 [01:00<00:03, 16.06 examples/s]
Tokenizing Prompts (num_proc=16): 100%|██████████| 1024/1024 [01:04<00:00, 16.32 examples/s]
Tokenizing Prompts (num_proc=16): 100%|██████████| 1024/1024 [01:05<00:00, 15.74 examples/s]
[2026-08-13 15:50:27,061] [INFO] [axolotl.utils.data.utils._log_dataset_stats:212] [PID:1747] min_input_len: 255
[2026-08-13 15:50:27,061] [INFO] [axolotl.utils.data.utils._log_dataset_stats:213] [PID:1747] max_input_len: 2091

Dropping Invalid Sequences (<None or >4096) (num_proc=16):   0%|          | 0/1024 [00:00<?, ? examples/s]
Dropping Invalid Sequences (<None or >4096) (num_proc=16):   6%|▋         | 64/1024 [00:00<00:05, 189.02 examples/s]
Dropping Invalid Sequences (<None or >4096) (num_proc=16): 100%|██████████| 1024/1024 [00:00<00:00, 1687.50 examples/s]

Saving the dataset (0/4 shards):   0%|          | 0/1024 [00:00<?, ? examples/s]
Saving the dataset (0/4 shards):  25%|██▌       | 256/1024 [00:06<00:20, 37.79 examples/s]
Saving the dataset (1/4 shards):  25%|██▌       | 256/1024 [00:06<00:20, 37.79 examples/s]
Saving the dataset (2/4 shards):  50%|█████     | 512/1024 [00:06<00:13, 37.79 examples/s]
Saving the dataset (3/4 shards):  75%|███████▌  | 768/1024 [00:06<00:06, 37.79 examples/s]
Saving the dataset (4/4 shards): 100%|██████████| 1024/1024 [00:06<00:00, 37.79 examples/s]
Saving the dataset (4/4 shards): 100%|██████████| 1024/1024 [00:07<00:00, 129.45 examples/s]
[2026-08-13 15:50:35,788] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:420] [PID:1747] total_num_tokens: 654_917
[2026-08-13 15:50:35,802] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:438] [PID:1747] `total_supervised_tokens: 236_779`
[2026-08-13 15:50:35,802] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:521] [PID:1747] total_num_steps: 128
[2026-08-13 15:50:35,802] [INFO] [axolotl.utils.data.sft._prepare_standard_dataset:121] [PID:1747] Maximum number of steps set at 128
[2026-08-13 15:50:35,803] [DEBUG] [axolotl.train.setup_model_and_tokenizer:70] [PID:1747] loading tokenizer... /workspace/python4-aft-v2-state/20260813T154138Z/control/parent/control/sft/end
[2026-08-13 15:50:37,630] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:311] [PID:1747] EOS: 106 / <end_of_turn>
[2026-08-13 15:50:37,630] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:312] [PID:1747] BOS: 2 / <bos>
[2026-08-13 15:50:37,630] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:313] [PID:1747] PAD: 0 / <pad>
[2026-08-13 15:50:37,630] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:314] [PID:1747] UNK: 3 / <unk>
[2026-08-13 15:50:39,540] [DEBUG] [axolotl.train.setup_model_and_tokenizer:81] [PID:1747] Loading model
[2026-08-13 15:50:39,545] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:75] [PID:1747] Patched OptimState8bit for torch.compile compatibility
[2026-08-13 15:50:39,545] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:122] [PID:1747] Patched OptimState4bit for torch.compile compatibility
[2026-08-13 15:50:39,545] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:154] [PID:1747] Patched OptimStateFp8 for torch.compile compatibility
[2026-08-13 15:50:39,550] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_evaluation_loop:94] [PID:1747] Patched Trainer.evaluation_loop with nanmean loss calculation
[2026-08-13 15:50:39,551] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_maybe_log_save_evaluate:148] [PID:1747] Patched Trainer._maybe_log_save_evaluate with nanmean loss calculation
[2026-08-13 15:50:39,621] [INFO] [axolotl.monkeypatch.attention.flash_attn_4.patch_flash_attn_4:52] [PID:1747] Flash Attention 4 is available for your GPU and offers faster training speeds. To enable: pip install flash-attn-4
[2026-08-13 15:50:40,840] [INFO] [axolotl.monkeypatch.lora_kernels.patch_self_attn_lora:304] [PID:1747] Patched attention class with LoRA optims: Gemma3Attention
[2026-08-13 15:50:41,076] [INFO] [axolotl.integrations.liger.plugin.pre_model_load:117] [PID:1747] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True}

Loading weights:   0%|          | 0/1248 [00:00<?, ?it/s]
Loading weights:  41%|████▏     | 516/1248 [00:00<00:00, 5149.35it/s]
Loading weights:  83%|████████▎ | 1031/1248 [00:00<00:00, 4747.31it/s]
Loading weights: 100%|██████████| 1248/1248 [00:00<00:00, 4958.27it/s]
[2026-08-13 15:50:45,860] [INFO] [axolotl.loaders.model._configure_embedding_dtypes:433] [PID:1747] Converting modules to torch.bfloat16
[2026-08-13 15:50:47,761] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:1747] Memory usage after model load 0.000GB ()
trainable params: 454,066,176 || all params: 27,886,472,816 || trainable%: 1.6283
[2026-08-13 15:50:50,831] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:1747] after adapters 0.000GB ()
[2026-08-13 15:50:58,971] [INFO] [axolotl.train.save_initial_configs:450] [PID:1747] Pre-saving adapter config to /workspace/python4-aft-v2-20260813T154138Z-control/experiments/python4/aft_v2/runs/20260813T154138Z/arms/control/train/checkpoints...
[2026-08-13 15:50:58,972] [INFO] [axolotl.train.save_initial_configs:454] [PID:1747] Pre-saving tokenizer to /workspace/python4-aft-v2-20260813T154138Z-control/experiments/python4/aft_v2/runs/20260813T154138Z/arms/control/train/checkpoints...
[2026-08-13 15:50:59,306] [INFO] [axolotl.train.save_initial_configs:459] [PID:1747] Pre-saving model config to /workspace/python4-aft-v2-20260813T154138Z-control/experiments/python4/aft_v2/runs/20260813T154138Z/arms/control/train/checkpoints...
[2026-08-13 15:50:59,309] [INFO] [axolotl.train.save_initial_configs:463] [PID:1747] Pre-saving processor to /workspace/python4-aft-v2-20260813T154138Z-control/experiments/python4/aft_v2/runs/20260813T154138Z/arms/control/train/checkpoints...
[2026-08-13 15:50:59,602] [INFO] [axolotl.train.execute_training:226] [PID:1747] Starting trainer...

  0%|          | 0/128 [00:00<?, ?it/s]
  1%|          | 1/128 [00:13<28:31, 13.48s/it]
                                               
{'loss': '0.8531', 'grad_norm': '1.081', 'learning_rate': '0', 'ppl': '2.347', 'memory/max_active (GiB)': '59.46', 'memory/max_allocated (GiB)': '59.46', 'memory/device_reserved (GiB)': '60.14', 'tokens/train_per_sec_per_gpu': '109.6', 'tokens/total': 28572, 'tokens/trainable': 7575, 'epoch': '0.03125'}

  1%|          | 1/128 [00:13<28:31, 13.48s/it]
  2%|▏         | 2/128 [00:23<24:25, 11.63s/it]
                                               
{'loss': '0.955', 'grad_norm': '0.9706', 'learning_rate': '1.667e-05', 'ppl': '2.599', 'memory/max_active (GiB)': '61.77', 'memory/max_allocated (GiB)': '61.77', 'memory/device_reserved (GiB)': '62.02', 'tokens/train_per_sec_per_gpu': '120.5', 'tokens/total': 56512, 'tokens/trainable': 15865, 'epoch': '0.0625'}

  2%|▏         | 2/128 [00:23<24:25, 11.63s/it]
  2%|▏         | 3/128 [00:33<22:00, 10.56s/it]
                                               
{'loss': '0.8412', 'grad_norm': '1.071', 'learning_rate': '3.333e-05', 'ppl': '2.319', 'memory/max_active (GiB)': '61.94', 'memory/max_allocated (GiB)': '61.94', 'memory/device_reserved (GiB)': '62.28', 'tokens/train_per_sec_per_gpu': '83.8', 'tokens/total': 82320, 'tokens/trainable': 23211, 'epoch': '0.09375'}

  2%|▏         | 3/128 [00:33<22:00, 10.56s/it]
  3%|▎         | 4/128 [00:43<22:00, 10.65s/it]
                                               
{'loss': '0.7129', 'grad_norm': '0.9637', 'learning_rate': '5e-05', 'ppl': '2.04', 'memory/max_active (GiB)': '62.39', 'memory/max_allocated (GiB)': '62.39', 'memory/device_reserved (GiB)': '62.74', 'tokens/train_per_sec_per_gpu': '85.97', 'tokens/total': 112500, 'tokens/trainable': 30046, 'epoch': '0.125'}

  3%|▎         | 4/128 [00:43<22:00, 10.65s/it]
  4%|▍         | 5/128 [00:54<21:35, 10.53s/it]
                                               
{'loss': '0.5368', 'grad_norm': '0.7934', 'learning_rate': '6.667e-05', 'ppl': '1.711', 'memory/max_active (GiB)': '65.24', 'memory/max_allocated (GiB)': '65.24', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '105.2', 'tokens/total': 142096, 'tokens/trainable': 37315, 'epoch': '0.1562'}

  4%|▍         | 5/128 [00:54<21:35, 10.53s/it]
  5%|▍         | 6/128 [01:03<20:52, 10.26s/it]
                                               
{'loss': '0.5395', 'grad_norm': '0.9188', 'learning_rate': '8.333e-05', 'ppl': '1.715', 'memory/max_active (GiB)': '62.16', 'memory/max_allocated (GiB)': '62.16', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '101.9', 'tokens/total': 168284, 'tokens/trainable': 44600, 'epoch': '0.1875'}

  5%|▍         | 6/128 [01:03<20:52, 10.26s/it]
  5%|▌         | 7/128 [01:13<20:32, 10.19s/it]
                                               
{'loss': '0.4757', 'grad_norm': '0.7304', 'learning_rate': '0.0001', 'ppl': '1.609', 'memory/max_active (GiB)': '61.52', 'memory/max_allocated (GiB)': '61.52', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '115.7', 'tokens/total': 196820, 'tokens/trainable': 52174, 'epoch': '0.2188'}

  5%|▌         | 7/128 [01:13<20:32, 10.19s/it]
  6%|▋         | 8/128 [01:24<20:33, 10.28s/it]
                                               
{'loss': '0.3853', 'grad_norm': '0.6223', 'learning_rate': '9.999e-05', 'ppl': '1.47', 'memory/max_active (GiB)': '62.62', 'memory/max_allocated (GiB)': '62.62', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '116.3', 'tokens/total': 226052, 'tokens/trainable': 59475, 'epoch': '0.25'}

  6%|▋         | 8/128 [01:24<20:33, 10.28s/it]
  7%|▋         | 9/128 [01:33<19:41,  9.93s/it]
                                               
{'loss': '0.3623', 'grad_norm': '0.4904', 'learning_rate': '9.994e-05', 'ppl': '1.437', 'memory/max_active (GiB)': '62.4', 'memory/max_allocated (GiB)': '62.4', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '72.07', 'tokens/total': 251360, 'tokens/trainable': 66221, 'epoch': '0.2812'}

  7%|▋         | 9/128 [01:33<19:41,  9.93s/it]
  8%|▊         | 10/128 [01:44<20:07, 10.23s/it]
                                                
{'loss': '0.3182', 'grad_norm': '0.4618', 'learning_rate': '9.987e-05', 'ppl': '1.375', 'memory/max_active (GiB)': '62.99', 'memory/max_allocated (GiB)': '62.99', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '105.7', 'tokens/total': 282048, 'tokens/trainable': 74909, 'epoch': '0.3125'}

  8%|▊         | 10/128 [01:44<20:07, 10.23s/it]
  9%|▊         | 11/128 [01:53<18:54,  9.70s/it]
                                                
{'loss': '0.2723', 'grad_norm': '0.4428', 'learning_rate': '9.976e-05', 'ppl': '1.313', 'memory/max_active (GiB)': '61.38', 'memory/max_allocated (GiB)': '61.38', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '67.89', 'tokens/total': 304988, 'tokens/trainable': 80926, 'epoch': '0.3438'}

  9%|▊         | 11/128 [01:53<18:54,  9.70s/it]
  9%|▉         | 12/128 [02:03<19:02,  9.85s/it]
                                                
{'loss': '0.397', 'grad_norm': '0.4366', 'learning_rate': '9.963e-05', 'ppl': '1.487', 'memory/max_active (GiB)': '61.88', 'memory/max_allocated (GiB)': '61.88', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '82.91', 'tokens/total': 333108, 'tokens/trainable': 87992, 'epoch': '0.375'}

  9%|▉         | 12/128 [02:03<19:02,  9.85s/it]
 10%|█         | 13/128 [02:12<18:25,  9.62s/it]
                                                
{'loss': '0.6852', 'grad_norm': '0.4224', 'learning_rate': '9.946e-05', 'ppl': '1.984', 'memory/max_active (GiB)': '60.88', 'memory/max_allocated (GiB)': '60.88', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '150.9', 'tokens/total': 356952, 'tokens/trainable': 96195, 'epoch': '0.4062'}

 10%|█         | 13/128 [02:12<18:25,  9.62s/it]
 11%|█         | 14/128 [02:21<18:17,  9.62s/it]
                                                
{'loss': '0.3385', 'grad_norm': '0.5395', 'learning_rate': '9.927e-05', 'ppl': '1.403', 'memory/max_active (GiB)': '61.4', 'memory/max_allocated (GiB)': '61.4', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '67.64', 'tokens/total': 384392, 'tokens/trainable': 103298, 'epoch': '0.4375'}

 11%|█         | 14/128 [02:21<18:17,  9.62s/it]
 12%|█▏        | 15/128 [02:32<18:33,  9.85s/it]
                                                
{'loss': '0.3099', 'grad_norm': '0.4232', 'learning_rate': '9.905e-05', 'ppl': '1.363', 'memory/max_active (GiB)': '62.24', 'memory/max_allocated (GiB)': '62.24', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '61.44', 'tokens/total': 413280, 'tokens/trainable': 109986, 'epoch': '0.4688'}

 12%|█▏        | 15/128 [02:32<18:33,  9.85s/it]
 12%|█▎        | 16/128 [02:42<18:46, 10.06s/it]
                                                
{'loss': '0.4417', 'grad_norm': '0.3693', 'learning_rate': '9.88e-05', 'ppl': '1.555', 'memory/max_active (GiB)': '62.53', 'memory/max_allocated (GiB)': '62.53', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '169.1', 'tokens/total': 443784, 'tokens/trainable': 118123, 'epoch': '0.5'}

 12%|█▎        | 16/128 [02:42<18:46, 10.06s/it]
 13%|█▎        | 17/128 [02:53<18:56, 10.24s/it]
                                                
{'loss': '0.3081', 'grad_norm': '0.3793', 'learning_rate': '9.852e-05', 'ppl': '1.361', 'memory/max_active (GiB)': '62.91', 'memory/max_allocated (GiB)': '62.91', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '110.3', 'tokens/total': 473580, 'tokens/trainable': 126313, 'epoch': '0.5312'}

 13%|█▎        | 17/128 [02:53<18:56, 10.24s/it]
 14%|█▍        | 18/128 [03:02<18:01,  9.83s/it]
                                                
{'loss': '0.2735', 'grad_norm': '0.4059', 'learning_rate': '9.821e-05', 'ppl': '1.315', 'memory/max_active (GiB)': '61.13', 'memory/max_allocated (GiB)': '61.13', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '111.3', 'tokens/total': 498056, 'tokens/trainable': 133442, 'epoch': '0.5625'}

 14%|█▍        | 18/128 [03:02<18:01,  9.83s/it]
 15%|█▍        | 19/128 [03:13<18:23, 10.12s/it]
                                                
{'loss': '0.2537', 'grad_norm': '0.3193', 'learning_rate': '9.787e-05', 'ppl': '1.289', 'memory/max_active (GiB)': '62.22', 'memory/max_allocated (GiB)': '62.22', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '55.72', 'tokens/total': 528120, 'tokens/trainable': 141989, 'epoch': '0.5938'}

 15%|█▍        | 19/128 [03:13<18:23, 10.12s/it]
 16%|█▌        | 20/128 [03:23<18:15, 10.15s/it]
                                                
{'loss': '0.394', 'grad_norm': '0.3583', 'learning_rate': '9.75e-05', 'ppl': '1.483', 'memory/max_active (GiB)': '62.31', 'memory/max_allocated (GiB)': '62.31', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '93.19', 'tokens/total': 557416, 'tokens/trainable': 149880, 'epoch': '0.625'}

 16%|█▌        | 20/128 [03:23<18:15, 10.15s/it]
 16%|█▋        | 21/128 [03:33<17:54, 10.04s/it]
                                                
{'loss': '0.297', 'grad_norm': '0.3307', 'learning_rate': '9.711e-05', 'ppl': '1.346', 'memory/max_active (GiB)': '61.34', 'memory/max_allocated (GiB)': '61.34', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '127.6', 'tokens/total': 583980, 'tokens/trainable': 158105, 'epoch': '0.6562'}

 16%|█▋        | 21/128 [03:33<17:54, 10.04s/it]
 17%|█▋        | 22/128 [03:42<17:06,  9.68s/it]
                                                
{'loss': '0.2395', 'grad_norm': '1.413', 'learning_rate': '9.668e-05', 'ppl': '1.271', 'memory/max_active (GiB)': '61.46', 'memory/max_allocated (GiB)': '61.46', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '120', 'tokens/total': 608384, 'tokens/trainable': 164670, 'epoch': '0.6875'}

 17%|█▋        | 22/128 [03:42<17:06,  9.68s/it]
 18%|█▊        | 23/128 [03:51<16:51,  9.63s/it]
                                                
{'loss': '0.233', 'grad_norm': '0.3677', 'learning_rate': '9.623e-05', 'ppl': '1.262', 'memory/max_active (GiB)': '61.08', 'memory/max_allocated (GiB)': '61.08', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '120.3', 'tokens/total': 634132, 'tokens/trainable': 171415, 'epoch': '0.7188'}

 18%|█▊        | 23/128 [03:51<16:51,  9.63s/it]
 19%|█▉        | 24/128 [04:00<16:08,  9.32s/it]
                                                
{'loss': '0.2242', 'grad_norm': '0.3338', 'learning_rate': '9.576e-05', 'ppl': '1.251', 'memory/max_active (GiB)': '61.04', 'memory/max_allocated (GiB)': '61.04', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '74.81', 'tokens/total': 657788, 'tokens/trainable': 177686, 'epoch': '0.75'}

 19%|█▉        | 24/128 [04:00<16:08,  9.32s/it]
 20%|█▉        | 25/128 [04:09<16:14,  9.46s/it]
                                                
{'loss': '0.2581', 'grad_norm': '0.3272', 'learning_rate': '9.525e-05', 'ppl': '1.294', 'memory/max_active (GiB)': '61.38', 'memory/max_allocated (GiB)': '61.38', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '82.8', 'tokens/total': 684376, 'tokens/trainable': 185346, 'epoch': '0.7812'}

 20%|█▉        | 25/128 [04:09<16:14,  9.46s/it]
 20%|██        | 26/128 [04:19<16:13,  9.55s/it]
                                                
{'loss': '0.3165', 'grad_norm': '0.3571', 'learning_rate': '9.472e-05', 'ppl': '1.372', 'memory/max_active (GiB)': '61.86', 'memory/max_allocated (GiB)': '61.86', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '64.77', 'tokens/total': 711992, 'tokens/trainable': 192605, 'epoch': '0.8125'}

 20%|██        | 26/128 [04:19<16:13,  9.55s/it]
 21%|██        | 27/128 [04:28<15:41,  9.32s/it]
                                                
{'loss': '0.2666', 'grad_norm': '0.3385', 'learning_rate': '9.416e-05', 'ppl': '1.306', 'memory/max_active (GiB)': '61.57', 'memory/max_allocated (GiB)': '61.57', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '69.54', 'tokens/total': 734816, 'tokens/trainable': 198978, 'epoch': '0.8438'}

 21%|██        | 27/128 [04:28<15:41,  9.32s/it]
 22%|██▏       | 28/128 [04:38<16:07,  9.68s/it]
                                                
{'loss': '0.2781', 'grad_norm': '0.3032', 'learning_rate': '9.358e-05', 'ppl': '1.321', 'memory/max_active (GiB)': '62.85', 'memory/max_allocated (GiB)': '62.85', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '126.9', 'tokens/total': 763996, 'tokens/trainable': 207138, 'epoch': '0.875'}

 22%|██▏       | 28/128 [04:38<16:07,  9.68s/it]
 23%|██▎       | 29/128 [04:48<16:07,  9.78s/it]
                                                
{'loss': '0.4731', 'grad_norm': '0.3914', 'learning_rate': '9.297e-05', 'ppl': '1.605', 'memory/max_active (GiB)': '63', 'memory/max_allocated (GiB)': '63', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '66.63', 'tokens/total': 792560, 'tokens/trainable': 214833, 'epoch': '0.9062'}

 23%|██▎       | 29/128 [04:48<16:07,  9.78s/it]
 23%|██▎       | 30/128 [04:58<15:57,  9.77s/it]
                                                
{'loss': '0.2909', 'grad_norm': '0.3517', 'learning_rate': '9.234e-05', 'ppl': '1.338', 'memory/max_active (GiB)': '61.8', 'memory/max_allocated (GiB)': '61.8', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '89.1', 'tokens/total': 818900, 'tokens/trainable': 222291, 'epoch': '0.9375'}

 23%|██▎       | 30/128 [04:58<15:57,  9.77s/it]
 24%|██▍       | 31/128 [05:08<15:39,  9.68s/it]
                                                
{'loss': '0.3248', 'grad_norm': '0.3643', 'learning_rate': '9.168e-05', 'ppl': '1.384', 'memory/max_active (GiB)': '61.47', 'memory/max_allocated (GiB)': '61.47', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '66.69', 'tokens/total': 845156, 'tokens/trainable': 229659, 'epoch': '0.9688'}

 24%|██▍       | 31/128 [05:08<15:39,  9.68s/it]
 25%|██▌       | 32/128 [05:17<15:30,  9.69s/it]
                                                
{'loss': '0.333', 'grad_norm': '0.3602', 'learning_rate': '9.099e-05', 'ppl': '1.395', 'memory/max_active (GiB)': '62.03', 'memory/max_allocated (GiB)': '62.03', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '87.12', 'tokens/total': 870564, 'tokens/trainable': 236779, 'epoch': '1'}

 25%|██▌       | 32/128 [05:17<15:30,  9.69s/it]
 26%|██▌       | 33/128 [05:27<15:21,  9.70s/it]
                                                
{'loss': '0.3966', 'grad_norm': '0.3184', 'learning_rate': '9.029e-05', 'ppl': '1.487', 'memory/max_active (GiB)': '61.23', 'memory/max_allocated (GiB)': '61.23', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '136.1', 'tokens/total': 894852, 'tokens/trainable': 244968, 'epoch': '1.031'}

 26%|██▌       | 33/128 [05:27<15:21,  9.70s/it]
 27%|██▋       | 34/128 [05:38<15:32,  9.92s/it]
                                                
{'loss': '0.184', 'grad_norm': '0.382', 'learning_rate': '8.955e-05', 'ppl': '1.202', 'memory/max_active (GiB)': '61.85', 'memory/max_allocated (GiB)': '61.85', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '104.7', 'tokens/total': 923880, 'tokens/trainable': 251510, 'epoch': '1.062'}

 27%|██▋       | 34/128 [05:38<15:32,  9.92s/it]
 27%|██▋       | 35/128 [05:47<15:14,  9.83s/it]
                                                
{'loss': '0.2092', 'grad_norm': '0.3009', 'learning_rate': '8.88e-05', 'ppl': '1.233', 'memory/max_active (GiB)': '61.83', 'memory/max_allocated (GiB)': '61.83', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '128.9', 'tokens/total': 951396, 'tokens/trainable': 258730, 'epoch': '1.094'}

 27%|██▋       | 35/128 [05:47<15:14,  9.83s/it]
 28%|██▊       | 36/128 [05:57<14:55,  9.73s/it]
                                                
{'loss': '0.2861', 'grad_norm': '0.3181', 'learning_rate': '8.802e-05', 'ppl': '1.331', 'memory/max_active (GiB)': '62.39', 'memory/max_allocated (GiB)': '62.39', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '67.11', 'tokens/total': 977128, 'tokens/trainable': 265887, 'epoch': '1.125'}

 28%|██▊       | 36/128 [05:57<14:55,  9.73s/it]
 29%|██▉       | 37/128 [06:06<14:37,  9.65s/it]
                                                
{'loss': '0.2319', 'grad_norm': '0.3034', 'learning_rate': '8.723e-05', 'ppl': '1.261', 'memory/max_active (GiB)': '61.45', 'memory/max_allocated (GiB)': '61.45', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '66.92', 'tokens/total': 1003672, 'tokens/trainable': 272560, 'epoch': '1.156'}

 29%|██▉       | 37/128 [06:06<14:37,  9.65s/it]
 30%|██▉       | 38/128 [06:16<14:42,  9.81s/it]
                                                
{'loss': '0.214', 'grad_norm': '0.3713', 'learning_rate': '8.641e-05', 'ppl': '1.239', 'memory/max_active (GiB)': '61.95', 'memory/max_allocated (GiB)': '61.95', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '120.6', 'tokens/total': 1031564, 'tokens/trainable': 280802, 'epoch': '1.188'}

 30%|██▉       | 38/128 [06:16<14:42,  9.81s/it]
 30%|███       | 39/128 [06:26<14:27,  9.74s/it]
                                                
{'loss': '0.2757', 'grad_norm': '0.3455', 'learning_rate': '8.557e-05', 'ppl': '1.317', 'memory/max_active (GiB)': '62.17', 'memory/max_allocated (GiB)': '62.17', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '140.9', 'tokens/total': 1058404, 'tokens/trainable': 288020, 'epoch': '1.219'}

 30%|███       | 39/128 [06:26<14:27,  9.74s/it]
 31%|███▏      | 40/128 [06:36<14:13,  9.70s/it]
                                                
{'loss': '0.2306', 'grad_norm': '0.3814', 'learning_rate': '8.471e-05', 'ppl': '1.259', 'memory/max_active (GiB)': '62.44', 'memory/max_allocated (GiB)': '62.44', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '94.66', 'tokens/total': 1084208, 'tokens/trainable': 295082, 'epoch': '1.25'}

 31%|███▏      | 40/128 [06:36<14:13,  9.70s/it]
 32%|███▏      | 41/128 [06:45<13:52,  9.57s/it]
                                                
{'loss': '0.1687', 'grad_norm': '0.3889', 'learning_rate': '8.383e-05', 'ppl': '1.184', 'memory/max_active (GiB)': '61.51', 'memory/max_allocated (GiB)': '61.51', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '91.4', 'tokens/total': 1110428, 'tokens/trainable': 301185, 'epoch': '1.281'}

 32%|███▏      | 41/128 [06:45<13:52,  9.57s/it]
 33%|███▎      | 42/128 [06:55<14:08,  9.86s/it]
                                                
{'loss': '0.2139', 'grad_norm': '0.3906', 'learning_rate': '8.293e-05', 'ppl': '1.238', 'memory/max_active (GiB)': '62.62', 'memory/max_allocated (GiB)': '62.62', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '95.87', 'tokens/total': 1139844, 'tokens/trainable': 309987, 'epoch': '1.312'}

 33%|███▎      | 42/128 [06:55<14:08,  9.86s/it]
 34%|███▎      | 43/128 [07:05<13:52,  9.79s/it]
                                                
{'loss': '0.1568', 'grad_norm': '0.4231', 'learning_rate': '8.201e-05', 'ppl': '1.17', 'memory/max_active (GiB)': '61.87', 'memory/max_allocated (GiB)': '61.87', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '72.42', 'tokens/total': 1167068, 'tokens/trainable': 317133, 'epoch': '1.344'}

 34%|███▎      | 43/128 [07:05<13:52,  9.79s/it]
 34%|███▍      | 44/128 [07:15<13:38,  9.74s/it]
                                                
{'loss': '0.2023', 'grad_norm': '0.3514', 'learning_rate': '8.107e-05', 'ppl': '1.224', 'memory/max_active (GiB)': '61.36', 'memory/max_allocated (GiB)': '61.36', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '97.61', 'tokens/total': 1192996, 'tokens/trainable': 324270, 'epoch': '1.375'}

 34%|███▍      | 44/128 [07:15<13:38,  9.74s/it]
 35%|███▌      | 45/128 [07:24<13:23,  9.68s/it]
                                                
{'loss': '0.2851', 'grad_norm': '0.3687', 'learning_rate': '8.012e-05', 'ppl': '1.33', 'memory/max_active (GiB)': '61.62', 'memory/max_allocated (GiB)': '61.62', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '105.6', 'tokens/total': 1218580, 'tokens/trainable': 331679, 'epoch': '1.406'}

 35%|███▌      | 45/128 [07:24<13:23,  9.68s/it]
 36%|███▌      | 46/128 [07:34<13:15,  9.70s/it]
                                                
{'loss': '0.2147', 'grad_norm': '0.4638', 'learning_rate': '7.915e-05', 'ppl': '1.239', 'memory/max_active (GiB)': '62.4', 'memory/max_allocated (GiB)': '62.4', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '84.11', 'tokens/total': 1246588, 'tokens/trainable': 339009, 'epoch': '1.438'}

 36%|███▌      | 46/128 [07:34<13:15,  9.70s/it]
 37%|███▋      | 47/128 [07:43<12:42,  9.41s/it]
                                                
{'loss': '0.2514', 'grad_norm': '0.3175', 'learning_rate': '7.817e-05', 'ppl': '1.286', 'memory/max_active (GiB)': '60.99', 'memory/max_allocated (GiB)': '60.99', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '120.4', 'tokens/total': 1269360, 'tokens/trainable': 346896, 'epoch': '1.469'}

 37%|███▋      | 47/128 [07:43<12:42,  9.41s/it]
 38%|███▊      | 48/128 [07:52<12:24,  9.30s/it]
                                                
{'loss': '0.2283', 'grad_norm': '0.4341', 'learning_rate': '7.716e-05', 'ppl': '1.256', 'memory/max_active (GiB)': '61.49', 'memory/max_allocated (GiB)': '61.49', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '72.94', 'tokens/total': 1293848, 'tokens/trainable': 354504, 'epoch': '1.5'}

 38%|███▊      | 48/128 [07:52<12:24,  9.30s/it]
 38%|███▊      | 49/128 [08:03<13:04,  9.93s/it]
                                                
{'loss': '0.195', 'grad_norm': '0.3058', 'learning_rate': '7.615e-05', 'ppl': '1.215', 'memory/max_active (GiB)': '62.99', 'memory/max_allocated (GiB)': '62.99', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '113.1', 'tokens/total': 1326248, 'tokens/trainable': 363672, 'epoch': '1.531'}

 38%|███▊      | 49/128 [08:03<13:04,  9.93s/it]
 39%|███▉      | 50/128 [08:12<12:34,  9.67s/it]
                                                
{'loss': '0.2623', 'grad_norm': '0.3641', 'learning_rate': '7.512e-05', 'ppl': '1.3', 'memory/max_active (GiB)': '61.29', 'memory/max_allocated (GiB)': '61.29', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '74.73', 'tokens/total': 1351092, 'tokens/trainable': 370812, 'epoch': '1.562'}

 39%|███▉      | 50/128 [08:12<12:34,  9.67s/it]
 40%|███▉      | 51/128 [08:21<12:10,  9.49s/it]
                                                
{'loss': '0.1637', 'grad_norm': '0.3236', 'learning_rate': '7.408e-05', 'ppl': '1.178', 'memory/max_active (GiB)': '61.36', 'memory/max_allocated (GiB)': '61.36', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '117', 'tokens/total': 1375188, 'tokens/trainable': 378084, 'epoch': '1.594'}

 40%|███▉      | 51/128 [08:21<12:10,  9.49s/it]
 41%|████      | 52/128 [08:31<12:09,  9.60s/it]
                                                
{'loss': '0.2152', 'grad_norm': '0.3374', 'learning_rate': '7.302e-05', 'ppl': '1.24', 'memory/max_active (GiB)': '62.85', 'memory/max_allocated (GiB)': '62.85', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '85.43', 'tokens/total': 1403484, 'tokens/trainable': 385450, 'epoch': '1.625'}

 41%|████      | 52/128 [08:31<12:09,  9.60s/it]
 41%|████▏     | 53/128 [08:41<12:08,  9.71s/it]
                                                
{'loss': '0.3259', 'grad_norm': '0.3772', 'learning_rate': '7.195e-05', 'ppl': '1.385', 'memory/max_active (GiB)': '61.45', 'memory/max_allocated (GiB)': '61.45', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '80.62', 'tokens/total': 1430556, 'tokens/trainable': 393200, 'epoch': '1.656'}

 41%|████▏     | 53/128 [08:41<12:08,  9.71s/it]
 42%|████▏     | 54/128 [08:52<12:36, 10.22s/it]
                                                
{'loss': '0.1534', 'grad_norm': '0.4093', 'learning_rate': '7.087e-05', 'ppl': '1.166', 'memory/max_active (GiB)': '62.53', 'memory/max_allocated (GiB)': '62.53', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '123.7', 'tokens/total': 1464160, 'tokens/trainable': 401245, 'epoch': '1.688'}

 42%|████▏     | 54/128 [08:52<12:36, 10.22s/it]
 43%|████▎     | 55/128 [09:03<12:26, 10.22s/it]
                                                
{'loss': '0.1612', 'grad_norm': '0.3807', 'learning_rate': '6.978e-05', 'ppl': '1.175', 'memory/max_active (GiB)': '61.87', 'memory/max_allocated (GiB)': '61.87', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '96.69', 'tokens/total': 1492720, 'tokens/trainable': 407748, 'epoch': '1.719'}

 43%|████▎     | 55/128 [09:03<12:26, 10.22s/it]
 44%|████▍     | 56/128 [09:12<11:59,  9.99s/it]
                                                
{'loss': '0.2231', 'grad_norm': '0.3648', 'learning_rate': '6.869e-05', 'ppl': '1.25', 'memory/max_active (GiB)': '62.91', 'memory/max_allocated (GiB)': '62.91', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '111.8', 'tokens/total': 1519496, 'tokens/trainable': 414529, 'epoch': '1.75'}

 44%|████▍     | 56/128 [09:12<11:59,  9.99s/it]
 45%|████▍     | 57/128 [09:22<11:43,  9.91s/it]
                                                
{'loss': '0.1865', 'grad_norm': '0.3956', 'learning_rate': '6.758e-05', 'ppl': '1.205', 'memory/max_active (GiB)': '61.92', 'memory/max_allocated (GiB)': '61.92', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '111.5', 'tokens/total': 1545740, 'tokens/trainable': 421282, 'epoch': '1.781'}

 45%|████▍     | 57/128 [09:22<11:43,  9.91s/it]
 45%|████▌     | 58/128 [09:31<11:28,  9.83s/it]
                                                
{'loss': '0.2678', 'grad_norm': '0.3839', 'learning_rate': '6.646e-05', 'ppl': '1.307', 'memory/max_active (GiB)': '61.42', 'memory/max_allocated (GiB)': '61.42', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '68.16', 'tokens/total': 1573476, 'tokens/trainable': 429253, 'epoch': '1.812'}

 45%|████▌     | 58/128 [09:31<11:28,  9.83s/it]
 46%|████▌     | 59/128 [09:41<11:21,  9.88s/it]
                                                
{'loss': '0.2599', 'grad_norm': '0.4043', 'learning_rate': '6.534e-05', 'ppl': '1.297', 'memory/max_active (GiB)': '62.38', 'memory/max_allocated (GiB)': '62.38', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '80.41', 'tokens/total': 1600756, 'tokens/trainable': 437205, 'epoch': '1.844'}

 46%|████▌     | 59/128 [09:41<11:21,  9.88s/it]
 47%|████▋     | 60/128 [09:52<11:26, 10.10s/it]
                                                
{'loss': '0.267', 'grad_norm': '0.3348', 'learning_rate': '6.42e-05', 'ppl': '1.306', 'memory/max_active (GiB)': '62.21', 'memory/max_allocated (GiB)': '62.21', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '141.5', 'tokens/total': 1631348, 'tokens/trainable': 445596, 'epoch': '1.875'}

 47%|████▋     | 60/128 [09:52<11:26, 10.10s/it]
 48%|████▊     | 61/128 [10:02<11:18, 10.12s/it]
                                                
{'loss': '0.154', 'grad_norm': '0.3754', 'learning_rate': '6.307e-05', 'ppl': '1.167', 'memory/max_active (GiB)': '62.84', 'memory/max_allocated (GiB)': '62.84', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '54.24', 'tokens/total': 1659460, 'tokens/trainable': 451831, 'epoch': '1.906'}

 48%|████▊     | 61/128 [10:02<11:18, 10.12s/it]
 48%|████▊     | 62/128 [10:11<10:42,  9.73s/it]
                                                
{'loss': '0.2324', 'grad_norm': '0.3766', 'learning_rate': '6.193e-05', 'ppl': '1.262', 'memory/max_active (GiB)': '61.16', 'memory/max_allocated (GiB)': '61.16', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '77.35', 'tokens/total': 1683816, 'tokens/trainable': 458074, 'epoch': '1.938'}

 48%|████▊     | 62/128 [10:11<10:42,  9.73s/it]
 49%|████▉     | 63/128 [10:20<10:25,  9.62s/it]
                                                
{'loss': '0.3328', 'grad_norm': '0.3646', 'learning_rate': '6.078e-05', 'ppl': '1.395', 'memory/max_active (GiB)': '61.46', 'memory/max_allocated (GiB)': '61.46', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '110.6', 'tokens/total': 1709084, 'tokens/trainable': 466953, 'epoch': '1.969'}

 49%|████▉     | 63/128 [10:20<10:25,  9.62s/it]
 50%|█████     | 64/128 [10:31<10:37,  9.95s/it]
                                                
{'loss': '0.1571', 'grad_norm': '0.5713', 'learning_rate': '5.963e-05', 'ppl': '1.17', 'memory/max_active (GiB)': '65.23', 'memory/max_allocated (GiB)': '65.23', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '64.85', 'tokens/total': 1739072, 'tokens/trainable': 473558, 'epoch': '2'}

 50%|█████     | 64/128 [10:31<10:37,  9.95s/it]
 51%|█████     | 65/128 [10:41<10:23,  9.90s/it]
                                                
{'loss': '0.14', 'grad_norm': '0.2865', 'learning_rate': '5.847e-05', 'ppl': '1.15', 'memory/max_active (GiB)': '61.13', 'memory/max_allocated (GiB)': '61.13', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '113.5', 'tokens/total': 1763640, 'tokens/trainable': 481217, 'epoch': '2.031'}

 51%|█████     | 65/128 [10:41<10:23,  9.90s/it]
 52%|█████▏    | 66/128 [10:52<10:29, 10.15s/it]
                                                
{'loss': '0.1009', 'grad_norm': '8.826', 'learning_rate': '5.732e-05', 'ppl': '1.106', 'memory/max_active (GiB)': '62.2', 'memory/max_allocated (GiB)': '62.2', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '90.7', 'tokens/total': 1793396, 'tokens/trainable': 487839, 'epoch': '2.062'}

 52%|█████▏    | 66/128 [10:52<10:29, 10.15s/it]
 52%|█████▏    | 67/128 [11:01<09:59,  9.83s/it]
                                                
{'loss': '0.12', 'grad_norm': '0.3392', 'learning_rate': '5.616e-05', 'ppl': '1.128', 'memory/max_active (GiB)': '61.78', 'memory/max_allocated (GiB)': '61.78', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '106.1', 'tokens/total': 1818448, 'tokens/trainable': 493902, 'epoch': '2.094'}

 52%|█████▏    | 67/128 [11:01<09:59,  9.83s/it]
 53%|█████▎    | 68/128 [11:12<10:23, 10.39s/it]
                                                
{'loss': '0.1153', 'grad_norm': '0.3403', 'learning_rate': '5.5e-05', 'ppl': '1.122', 'memory/max_active (GiB)': '65.25', 'memory/max_allocated (GiB)': '65.25', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '73.83', 'tokens/total': 1852036, 'tokens/trainable': 502094, 'epoch': '2.125'}

 53%|█████▎    | 68/128 [11:12<10:23, 10.39s/it]
 54%|█████▍    | 69/128 [11:23<10:06, 10.29s/it]
                                                
{'loss': '0.1463', 'grad_norm': '0.3567', 'learning_rate': '5.384e-05', 'ppl': '1.158', 'memory/max_active (GiB)': '61.71', 'memory/max_allocated (GiB)': '61.71', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '101.1', 'tokens/total': 1880656, 'tokens/trainable': 510593, 'epoch': '2.156'}

 54%|█████▍    | 69/128 [11:23<10:06, 10.29s/it]
 55%|█████▍    | 70/128 [11:32<09:37,  9.95s/it]
                                                
{'loss': '0.1803', 'grad_norm': '0.4147', 'learning_rate': '5.268e-05', 'ppl': '1.198', 'memory/max_active (GiB)': '61.28', 'memory/max_allocated (GiB)': '61.28', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '112.1', 'tokens/total': 1905028, 'tokens/trainable': 518250, 'epoch': '2.188'}

 55%|█████▍    | 70/128 [11:32<09:37,  9.95s/it]
 55%|█████▌    | 71/128 [11:40<09:07,  9.61s/it]
                                                
{'loss': '0.2213', 'grad_norm': '0.577', 'learning_rate': '5.153e-05', 'ppl': '1.248', 'memory/max_active (GiB)': '61.1', 'memory/max_allocated (GiB)': '61.1', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '119.4', 'tokens/total': 1929140, 'tokens/trainable': 526478, 'epoch': '2.219'}

 55%|█████▌    | 71/128 [11:40<09:07,  9.61s/it]
 56%|█████▋    | 72/128 [11:50<08:53,  9.53s/it]
                                                
{'loss': '0.1114', 'grad_norm': '0.5331', 'learning_rate': '5.037e-05', 'ppl': '1.118', 'memory/max_active (GiB)': '60.92', 'memory/max_allocated (GiB)': '60.92', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '61.4', 'tokens/total': 1953896, 'tokens/trainable': 533140, 'epoch': '2.25'}

 56%|█████▋    | 72/128 [11:50<08:53,  9.53s/it]
 57%|█████▋    | 73/128 [11:59<08:43,  9.52s/it]
                                                
{'loss': '0.1008', 'grad_norm': '0.5824', 'learning_rate': '4.922e-05', 'ppl': '1.106', 'memory/max_active (GiB)': '60.96', 'memory/max_allocated (GiB)': '60.96', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '78.79', 'tokens/total': 1980148, 'tokens/trainable': 539469, 'epoch': '2.281'}

 57%|█████▋    | 73/128 [11:59<08:43,  9.52s/it]
 58%|█████▊    | 74/128 [12:09<08:28,  9.42s/it]
                                                
{'loss': '0.1759', 'grad_norm': '0.5028', 'learning_rate': '4.807e-05', 'ppl': '1.192', 'memory/max_active (GiB)': '61.59', 'memory/max_allocated (GiB)': '61.59', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '100.1', 'tokens/total': 2004292, 'tokens/trainable': 546977, 'epoch': '2.312'}

 58%|█████▊    | 74/128 [12:09<08:28,  9.42s/it]
 59%|█████▊    | 75/128 [12:19<08:32,  9.66s/it]
                                                
{'loss': '0.1146', 'grad_norm': '0.5454', 'learning_rate': '4.693e-05', 'ppl': '1.121', 'memory/max_active (GiB)': '62.3', 'memory/max_allocated (GiB)': '62.3', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '76.08', 'tokens/total': 2032520, 'tokens/trainable': 553216, 'epoch': '2.344'}

 59%|█████▊    | 75/128 [12:19<08:32,  9.66s/it]
 59%|█████▉    | 76/128 [12:30<08:44, 10.09s/it]
                                                
{'loss': '0.1039', 'grad_norm': '0.412', 'learning_rate': '4.58e-05', 'ppl': '1.11', 'memory/max_active (GiB)': '62.53', 'memory/max_allocated (GiB)': '62.53', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '108.9', 'tokens/total': 2064644, 'tokens/trainable': 560800, 'epoch': '2.375'}

 59%|█████▉    | 76/128 [12:30<08:44, 10.09s/it]
 60%|██████    | 77/128 [12:39<08:27,  9.96s/it]
                                                
{'loss': '0.2105', 'grad_norm': '0.453', 'learning_rate': '4.466e-05', 'ppl': '1.234', 'memory/max_active (GiB)': '61.61', 'memory/max_allocated (GiB)': '61.61', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '84.94', 'tokens/total': 2090812, 'tokens/trainable': 567665, 'epoch': '2.406'}

 60%|██████    | 77/128 [12:39<08:27,  9.96s/it]
 61%|██████    | 78/128 [12:50<08:25, 10.10s/it]
                                                
{'loss': '0.1437', 'grad_norm': '0.3885', 'learning_rate': '4.354e-05', 'ppl': '1.154', 'memory/max_active (GiB)': '63.01', 'memory/max_allocated (GiB)': '63.01', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '105.5', 'tokens/total': 2121540, 'tokens/trainable': 575399, 'epoch': '2.438'}

 61%|██████    | 78/128 [12:50<08:25, 10.10s/it]
 62%|██████▏   | 79/128 [12:59<08:04,  9.88s/it]
                                                
{'loss': '0.1632', 'grad_norm': '0.5362', 'learning_rate': '4.242e-05', 'ppl': '1.177', 'memory/max_active (GiB)': '61.05', 'memory/max_allocated (GiB)': '61.05', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '144.5', 'tokens/total': 2146788, 'tokens/trainable': 582816, 'epoch': '2.469'}

 62%|██████▏   | 79/128 [12:59<08:04,  9.88s/it]
 62%|██████▎   | 80/128 [13:09<07:46,  9.73s/it]
                                                
{'loss': '0.1305', 'grad_norm': '0.594', 'learning_rate': '4.131e-05', 'ppl': '1.139', 'memory/max_active (GiB)': '62.4', 'memory/max_allocated (GiB)': '62.4', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '105.4', 'tokens/total': 2173012, 'tokens/trainable': 590228, 'epoch': '2.5'}

 62%|██████▎   | 80/128 [13:09<07:46,  9.73s/it]
 63%|██████▎   | 81/128 [13:19<07:49, 10.00s/it]
                                                
{'loss': '0.1375', 'grad_norm': '0.4106', 'learning_rate': '4.022e-05', 'ppl': '1.147', 'memory/max_active (GiB)': '62.43', 'memory/max_allocated (GiB)': '62.43', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '103.7', 'tokens/total': 2202820, 'tokens/trainable': 597439, 'epoch': '2.531'}

 63%|██████▎   | 81/128 [13:19<07:49, 10.00s/it]
 64%|██████▍   | 82/128 [13:29<07:35,  9.90s/it]
                                                
{'loss': '0.2388', 'grad_norm': '0.6554', 'learning_rate': '3.913e-05', 'ppl': '1.27', 'memory/max_active (GiB)': '61.94', 'memory/max_allocated (GiB)': '61.94', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '114.6', 'tokens/total': 2230636, 'tokens/trainable': 606167, 'epoch': '2.562'}

 64%|██████▍   | 82/128 [13:29<07:35,  9.90s/it]
 65%|██████▍   | 83/128 [13:39<07:23,  9.85s/it]
                                                
{'loss': '0.0951', 'grad_norm': '0.4243', 'learning_rate': '3.805e-05', 'ppl': '1.1', 'memory/max_active (GiB)': '61.33', 'memory/max_allocated (GiB)': '61.33', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '89.33', 'tokens/total': 2257360, 'tokens/trainable': 613034, 'epoch': '2.594'}

 65%|██████▍   | 83/128 [13:39<07:23,  9.85s/it]
 66%|██████▌   | 84/128 [13:48<07:08,  9.73s/it]
                                                
{'loss': '0.1324', 'grad_norm': '0.3982', 'learning_rate': '3.698e-05', 'ppl': '1.142', 'memory/max_active (GiB)': '62.98', 'memory/max_allocated (GiB)': '62.98', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '84.23', 'tokens/total': 2284004, 'tokens/trainable': 620094, 'epoch': '2.625'}

 66%|██████▌   | 84/128 [13:48<07:08,  9.73s/it]
 66%|██████▋   | 85/128 [13:57<06:52,  9.59s/it]
                                                
{'loss': '0.1067', 'grad_norm': '0.4646', 'learning_rate': '3.592e-05', 'ppl': '1.113', 'memory/max_active (GiB)': '61.19', 'memory/max_allocated (GiB)': '61.19', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '127.5', 'tokens/total': 2308528, 'tokens/trainable': 626836, 'epoch': '2.656'}

 66%|██████▋   | 85/128 [13:57<06:52,  9.59s/it]
 67%|██████▋   | 86/128 [14:08<06:58,  9.96s/it]
                                                
{'loss': '0.1931', 'grad_norm': '1.086', 'learning_rate': '3.488e-05', 'ppl': '1.213', 'memory/max_active (GiB)': '62.85', 'memory/max_allocated (GiB)': '62.85', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '104.9', 'tokens/total': 2340360, 'tokens/trainable': 635364, 'epoch': '2.688'}

 67%|██████▋   | 86/128 [14:08<06:58,  9.96s/it]
 68%|██████▊   | 87/128 [14:17<06:35,  9.66s/it]
                                                
{'loss': '0.1149', 'grad_norm': '0.4243', 'learning_rate': '3.385e-05', 'ppl': '1.122', 'memory/max_active (GiB)': '61.33', 'memory/max_allocated (GiB)': '61.33', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '128.9', 'tokens/total': 2364116, 'tokens/trainable': 643111, 'epoch': '2.719'}

 68%|██████▊   | 87/128 [14:17<06:35,  9.66s/it]
 69%|██████▉   | 88/128 [14:27<06:24,  9.61s/it]
                                                
{'loss': '0.1185', 'grad_norm': '5.101', 'learning_rate': '3.284e-05', 'ppl': '1.126', 'memory/max_active (GiB)': '61.39', 'memory/max_allocated (GiB)': '61.39', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '123.2', 'tokens/total': 2390936, 'tokens/trainable': 650643, 'epoch': '2.75'}

 69%|██████▉   | 88/128 [14:27<06:24,  9.61s/it]
 70%|██████▉   | 89/128 [14:37<06:27,  9.93s/it]
                                                
{'loss': '0.1331', 'grad_norm': '0.4176', 'learning_rate': '3.183e-05', 'ppl': '1.142', 'memory/max_active (GiB)': '62.93', 'memory/max_allocated (GiB)': '62.93', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '119.3', 'tokens/total': 2421276, 'tokens/trainable': 658072, 'epoch': '2.781'}

 70%|██████▉   | 89/128 [14:37<06:27,  9.93s/it]
 70%|███████   | 90/128 [14:48<06:27, 10.19s/it]
                                                
{'loss': '0.1038', 'grad_norm': '0.3559', 'learning_rate': '3.085e-05', 'ppl': '1.109', 'memory/max_active (GiB)': '62.61', 'memory/max_allocated (GiB)': '62.61', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '72.36', 'tokens/total': 2452932, 'tokens/trainable': 665766, 'epoch': '2.812'}

 70%|███████   | 90/128 [14:48<06:27, 10.19s/it]
 71%|███████   | 91/128 [14:58<06:11, 10.05s/it]
                                                
{'loss': '0.09136', 'grad_norm': '0.3679', 'learning_rate': '2.988e-05', 'ppl': '1.096', 'memory/max_active (GiB)': '61.91', 'memory/max_allocated (GiB)': '61.91', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '71.83', 'tokens/total': 2479584, 'tokens/trainable': 672013, 'epoch': '2.844'}

 71%|███████   | 91/128 [14:58<06:11, 10.05s/it]
 72%|███████▏  | 92/128 [15:08<06:01, 10.05s/it]
                                                
{'loss': '0.1198', 'grad_norm': '0.4662', 'learning_rate': '2.893e-05', 'ppl': '1.127', 'memory/max_active (GiB)': '62.03', 'memory/max_allocated (GiB)': '62.03', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '85.16', 'tokens/total': 2506956, 'tokens/trainable': 679699, 'epoch': '2.875'}

 72%|███████▏  | 92/128 [15:08<06:01, 10.05s/it]
 73%|███████▎  | 93/128 [15:19<05:59, 10.28s/it]
                                                
{'loss': '0.1215', 'grad_norm': '0.4302', 'learning_rate': '2.799e-05', 'ppl': '1.129', 'memory/max_active (GiB)': '62.39', 'memory/max_allocated (GiB)': '62.39', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '86.66', 'tokens/total': 2538408, 'tokens/trainable': 686967, 'epoch': '2.906'}

 73%|███████▎  | 93/128 [15:19<05:59, 10.28s/it]
 73%|███████▎  | 94/128 [15:29<05:47, 10.22s/it]
                                                
{'loss': '0.1434', 'grad_norm': '0.5151', 'learning_rate': '2.707e-05', 'ppl': '1.154', 'memory/max_active (GiB)': '62.03', 'memory/max_allocated (GiB)': '62.03', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '65.75', 'tokens/total': 2566044, 'tokens/trainable': 695138, 'epoch': '2.938'}

 73%|███████▎  | 94/128 [15:29<05:47, 10.22s/it]
 74%|███████▍  | 95/128 [15:38<05:24,  9.84s/it]
                                                
{'loss': '0.1059', 'grad_norm': '0.4741', 'learning_rate': '2.617e-05', 'ppl': '1.112', 'memory/max_active (GiB)': '61.47', 'memory/max_allocated (GiB)': '61.47', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '153', 'tokens/total': 2591040, 'tokens/trainable': 702275, 'epoch': '2.969'}

 74%|███████▍  | 95/128 [15:38<05:24,  9.84s/it]
 75%|███████▌  | 96/128 [15:48<05:13,  9.81s/it]
                                                
{'loss': '0.1591', 'grad_norm': '0.4416', 'learning_rate': '2.529e-05', 'ppl': '1.173', 'memory/max_active (GiB)': '61.55', 'memory/max_allocated (GiB)': '61.55', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '115.7', 'tokens/total': 2616776, 'tokens/trainable': 710337, 'epoch': '3'}

 75%|███████▌  | 96/128 [15:48<05:13,  9.81s/it]
 76%|███████▌  | 97/128 [15:58<05:13, 10.10s/it]
                                                
{'loss': '0.096', 'grad_norm': '0.3499', 'learning_rate': '2.443e-05', 'ppl': '1.101', 'memory/max_active (GiB)': '62.44', 'memory/max_allocated (GiB)': '62.44', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '125.1', 'tokens/total': 2645272, 'tokens/trainable': 717607, 'epoch': '3.031'}

 76%|███████▌  | 97/128 [15:58<05:13, 10.10s/it]
 77%|███████▋  | 98/128 [16:08<05:00, 10.02s/it]
                                                
{'loss': '0.1012', 'grad_norm': '0.4029', 'learning_rate': '2.359e-05', 'ppl': '1.107', 'memory/max_active (GiB)': '61.86', 'memory/max_allocated (GiB)': '61.86', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '110', 'tokens/total': 2671656, 'tokens/trainable': 725080, 'epoch': '3.062'}

 77%|███████▋  | 98/128 [16:08<05:00, 10.02s/it]
 77%|███████▋  | 99/128 [16:17<04:39,  9.64s/it]
                                                
{'loss': '0.07406', 'grad_norm': '0.6041', 'learning_rate': '2.277e-05', 'ppl': '1.077', 'memory/max_active (GiB)': '61.11', 'memory/max_allocated (GiB)': '61.11', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '111.3', 'tokens/total': 2695780, 'tokens/trainable': 732499, 'epoch': '3.094'}

 77%|███████▋  | 99/128 [16:17<04:39,  9.64s/it]
 78%|███████▊  | 100/128 [16:26<04:25,  9.49s/it]
                                                 
{'loss': '0.07486', 'grad_norm': '0.6385', 'learning_rate': '2.198e-05', 'ppl': '1.078', 'memory/max_active (GiB)': '61.7', 'memory/max_allocated (GiB)': '61.7', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '86.9', 'tokens/total': 2720188, 'tokens/trainable': 739851, 'epoch': '3.125'}

 78%|███████▊  | 100/128 [16:26<04:25,  9.49s/it]
 79%|███████▉  | 101/128 [16:37<04:26,  9.89s/it]
                                                 
{'loss': '0.07097', 'grad_norm': '0.3953', 'learning_rate': '2.12e-05', 'ppl': '1.074', 'memory/max_active (GiB)': '62.84', 'memory/max_allocated (GiB)': '62.84', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '62.78', 'tokens/total': 2751896, 'tokens/trainable': 747545, 'epoch': '3.156'}

 79%|███████▉  | 101/128 [16:37<04:26,  9.89s/it]
 80%|███████▉  | 102/128 [16:46<04:14,  9.80s/it]
                                                 
{'loss': '0.1206', 'grad_norm': '0.4047', 'learning_rate': '2.045e-05', 'ppl': '1.128', 'memory/max_active (GiB)': '62.03', 'memory/max_allocated (GiB)': '62.03', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '72.46', 'tokens/total': 2777824, 'tokens/trainable': 754408, 'epoch': '3.188'}

 80%|███████▉  | 102/128 [16:46<04:14,  9.80s/it]
 80%|████████  | 103/128 [16:56<04:03,  9.75s/it]
                                                 
{'loss': '0.0522', 'grad_norm': '0.432', 'learning_rate': '1.971e-05', 'ppl': '1.054', 'memory/max_active (GiB)': '61.42', 'memory/max_allocated (GiB)': '61.42', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '86.5', 'tokens/total': 2805324, 'tokens/trainable': 760831, 'epoch': '3.219'}

 80%|████████  | 103/128 [16:56<04:03,  9.75s/it]
 81%|████████▏ | 104/128 [17:06<03:54,  9.75s/it]
                                                 
{'loss': '0.07371', 'grad_norm': '0.3952', 'learning_rate': '1.901e-05', 'ppl': '1.077', 'memory/max_active (GiB)': '61.83', 'memory/max_allocated (GiB)': '61.83', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '136.9', 'tokens/total': 2832068, 'tokens/trainable': 769191, 'epoch': '3.25'}

 81%|████████▏ | 104/128 [17:06<03:54,  9.75s/it]
 82%|████████▏ | 105/128 [17:15<03:43,  9.72s/it]
                                                 
{'loss': '0.07041', 'grad_norm': '0.5411', 'learning_rate': '1.832e-05', 'ppl': '1.073', 'memory/max_active (GiB)': '61.4', 'memory/max_allocated (GiB)': '61.4', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '71.05', 'tokens/total': 2859496, 'tokens/trainable': 776065, 'epoch': '3.281'}

 82%|████████▏ | 105/128 [17:15<03:43,  9.72s/it]
 83%|████████▎ | 106/128 [17:26<03:38,  9.93s/it]
                                                 
{'loss': '0.09393', 'grad_norm': '0.6271', 'learning_rate': '1.766e-05', 'ppl': '1.098', 'memory/max_active (GiB)': '62.4', 'memory/max_allocated (GiB)': '62.4', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '91.2', 'tokens/total': 2888168, 'tokens/trainable': 783919, 'epoch': '3.312'}

 83%|████████▎ | 106/128 [17:26<03:38,  9.93s/it]
 84%|████████▎ | 107/128 [17:37<03:32, 10.14s/it]
                                                 
{'loss': '0.05329', 'grad_norm': '1.391', 'learning_rate': '1.703e-05', 'ppl': '1.055', 'memory/max_active (GiB)': '63', 'memory/max_allocated (GiB)': '63', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '91.12', 'tokens/total': 2919108, 'tokens/trainable': 791343, 'epoch': '3.344'}

 84%|████████▎ | 107/128 [17:37<03:32, 10.14s/it]
 84%|████████▍ | 108/128 [17:47<03:22, 10.11s/it]
                                                 
{'loss': '0.07967', 'grad_norm': '0.6436', 'learning_rate': '1.642e-05', 'ppl': '1.083', 'memory/max_active (GiB)': '62.2', 'memory/max_allocated (GiB)': '62.2', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '105.7', 'tokens/total': 2946728, 'tokens/trainable': 799664, 'epoch': '3.375'}

 84%|████████▍ | 108/128 [17:47<03:22, 10.11s/it]
 85%|████████▌ | 109/128 [17:56<03:06,  9.84s/it]
                                                 
{'loss': '0.08347', 'grad_norm': '0.6207', 'learning_rate': '1.584e-05', 'ppl': '1.087', 'memory/max_active (GiB)': '62.24', 'memory/max_allocated (GiB)': '62.24', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '134.9', 'tokens/total': 2971256, 'tokens/trainable': 808020, 'epoch': '3.406'}

 85%|████████▌ | 109/128 [17:56<03:06,  9.84s/it]
 86%|████████▌ | 110/128 [18:05<02:54,  9.72s/it]
                                                 
{'loss': '0.09613', 'grad_norm': '0.4698', 'learning_rate': '1.528e-05', 'ppl': '1.101', 'memory/max_active (GiB)': '62.98', 'memory/max_allocated (GiB)': '62.98', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '141.7', 'tokens/total': 2998056, 'tokens/trainable': 816376, 'epoch': '3.438'}

 86%|████████▌ | 110/128 [18:05<02:54,  9.72s/it]
 87%|████████▋ | 111/128 [18:15<02:45,  9.76s/it]
                                                 
{'loss': '0.0865', 'grad_norm': '0.7513', 'learning_rate': '1.475e-05', 'ppl': '1.09', 'memory/max_active (GiB)': '61.46', 'memory/max_allocated (GiB)': '61.46', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '98.99', 'tokens/total': 3025116, 'tokens/trainable': 824339, 'epoch': '3.469'}

 87%|████████▋ | 111/128 [18:15<02:45,  9.76s/it]
 88%|████████▊ | 112/128 [18:25<02:35,  9.69s/it]
                                                 
{'loss': '0.07029', 'grad_norm': '0.6626', 'learning_rate': '1.424e-05', 'ppl': '1.073', 'memory/max_active (GiB)': '61.77', 'memory/max_allocated (GiB)': '61.77', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '89.84', 'tokens/total': 3051800, 'tokens/trainable': 831162, 'epoch': '3.5'}

 88%|████████▊ | 112/128 [18:25<02:35,  9.69s/it]
 88%|████████▊ | 113/128 [18:34<02:24,  9.65s/it]
                                                 
{'loss': '0.05055', 'grad_norm': '2.803', 'learning_rate': '1.377e-05', 'ppl': '1.052', 'memory/max_active (GiB)': '61.58', 'memory/max_allocated (GiB)': '61.58', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '91.22', 'tokens/total': 3077536, 'tokens/trainable': 837060, 'epoch': '3.531'}

 88%|████████▊ | 113/128 [18:34<02:24,  9.65s/it]
 89%|████████▉ | 114/128 [18:43<02:11,  9.41s/it]
                                                 
{'loss': '0.08243', 'grad_norm': '0.5624', 'learning_rate': '1.332e-05', 'ppl': '1.086', 'memory/max_active (GiB)': '61.22', 'memory/max_allocated (GiB)': '61.22', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '152.9', 'tokens/total': 3101764, 'tokens/trainable': 845319, 'epoch': '3.562'}

 89%|████████▉ | 114/128 [18:43<02:11,  9.41s/it]
 90%|████████▉ | 115/128 [18:52<02:01,  9.31s/it]
                                                 
{'loss': '0.06347', 'grad_norm': '0.5146', 'learning_rate': '1.289e-05', 'ppl': '1.066', 'memory/max_active (GiB)': '61.61', 'memory/max_allocated (GiB)': '61.61', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '62.03', 'tokens/total': 3125720, 'tokens/trainable': 852314, 'epoch': '3.594'}

 90%|████████▉ | 115/128 [18:52<02:01,  9.31s/it]
 91%|█████████ | 116/128 [19:01<01:51,  9.32s/it]
                                                 
{'loss': '0.05762', 'grad_norm': '0.9053', 'learning_rate': '1.25e-05', 'ppl': '1.059', 'memory/max_active (GiB)': '61.85', 'memory/max_allocated (GiB)': '61.85', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '84.63', 'tokens/total': 3152324, 'tokens/trainable': 858307, 'epoch': '3.625'}

 91%|█████████ | 116/128 [19:01<01:51,  9.32s/it]
 91%|█████████▏| 117/128 [19:11<01:44,  9.49s/it]
                                                 
{'loss': '0.0596', 'grad_norm': '0.4919', 'learning_rate': '1.213e-05', 'ppl': '1.061', 'memory/max_active (GiB)': '62.94', 'memory/max_allocated (GiB)': '62.94', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '150.4', 'tokens/total': 3179492, 'tokens/trainable': 865234, 'epoch': '3.656'}

 91%|█████████▏| 117/128 [19:11<01:44,  9.49s/it]
 92%|█████████▏| 118/128 [19:20<01:33,  9.37s/it]
                                                 
{'loss': '0.05849', 'grad_norm': '0.5534', 'learning_rate': '1.179e-05', 'ppl': '1.06', 'memory/max_active (GiB)': '61.06', 'memory/max_allocated (GiB)': '61.06', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '80.98', 'tokens/total': 3204652, 'tokens/trainable': 871968, 'epoch': '3.688'}

 92%|█████████▏| 118/128 [19:20<01:33,  9.37s/it]
 93%|█████████▎| 119/128 [19:31<01:27,  9.69s/it]
                                                 
{'loss': '0.08361', 'grad_norm': '0.4703', 'learning_rate': '1.148e-05', 'ppl': '1.087', 'memory/max_active (GiB)': '62.16', 'memory/max_allocated (GiB)': '62.16', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '57.35', 'tokens/total': 3233940, 'tokens/trainable': 878865, 'epoch': '3.719'}

 93%|█████████▎| 119/128 [19:31<01:27,  9.69s/it]
 94%|█████████▍| 120/128 [19:41<01:18,  9.79s/it]
                                                 
{'loss': '0.05424', 'grad_norm': '0.6142', 'learning_rate': '1.12e-05', 'ppl': '1.056', 'memory/max_active (GiB)': '61.76', 'memory/max_allocated (GiB)': '61.76', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '108.8', 'tokens/total': 3262500, 'tokens/trainable': 886214, 'epoch': '3.75'}

 94%|█████████▍| 120/128 [19:41<01:18,  9.79s/it]
 95%|█████████▍| 121/128 [19:51<01:10, 10.04s/it]
                                                 
{'loss': '0.06502', 'grad_norm': '0.6175', 'learning_rate': '1.095e-05', 'ppl': '1.067', 'memory/max_active (GiB)': '62.61', 'memory/max_allocated (GiB)': '62.61', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '122.5', 'tokens/total': 3292288, 'tokens/trainable': 893566, 'epoch': '3.781'}

 95%|█████████▍| 121/128 [19:51<01:10, 10.04s/it]
 95%|█████████▌| 122/128 [20:01<00:59,  9.92s/it]
                                                 
{'loss': '0.1094', 'grad_norm': '0.4898', 'learning_rate': '1.073e-05', 'ppl': '1.116', 'memory/max_active (GiB)': '61.17', 'memory/max_allocated (GiB)': '61.17', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '73.68', 'tokens/total': 3318256, 'tokens/trainable': 901937, 'epoch': '3.812'}

 95%|█████████▌| 122/128 [20:01<00:59,  9.92s/it]
 96%|█████████▌| 123/128 [20:10<00:48,  9.74s/it]
                                                 
{'loss': '0.05936', 'grad_norm': '0.484', 'learning_rate': '1.054e-05', 'ppl': '1.061', 'memory/max_active (GiB)': '62.91', 'memory/max_allocated (GiB)': '62.91', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '56.92', 'tokens/total': 3344436, 'tokens/trainable': 909008, 'epoch': '3.844'}

 96%|█████████▌| 123/128 [20:10<00:48,  9.74s/it]
 97%|█████████▋| 124/128 [20:20<00:39,  9.80s/it]
                                                 
{'loss': '0.05095', 'grad_norm': '0.4998', 'learning_rate': '1.037e-05', 'ppl': '1.052', 'memory/max_active (GiB)': '62.39', 'memory/max_allocated (GiB)': '62.39', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '68.78', 'tokens/total': 3371716, 'tokens/trainable': 915519, 'epoch': '3.875'}

 97%|█████████▋| 124/128 [20:20<00:39,  9.80s/it]
 98%|█████████▊| 125/128 [20:32<00:30, 10.21s/it]
                                                 
{'loss': '0.05779', 'grad_norm': '0.3826', 'learning_rate': '1.024e-05', 'ppl': '1.059', 'memory/max_active (GiB)': '62.85', 'memory/max_allocated (GiB)': '62.85', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '162.6', 'tokens/total': 3404300, 'tokens/trainable': 924818, 'epoch': '3.906'}

 98%|█████████▊| 125/128 [20:32<00:30, 10.21s/it]
 98%|█████████▊| 126/128 [20:42<00:20, 10.31s/it]
                                                 
{'loss': '0.06658', 'grad_norm': '0.5044', 'learning_rate': '1.013e-05', 'ppl': '1.069', 'memory/max_active (GiB)': '62.31', 'memory/max_allocated (GiB)': '62.31', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '112.4', 'tokens/total': 3434008, 'tokens/trainable': 931637, 'epoch': '3.938'}

 98%|█████████▊| 126/128 [20:42<00:20, 10.31s/it]
 99%|█████████▉| 127/128 [20:52<00:10, 10.33s/it]
                                                 
{'loss': '0.08437', 'grad_norm': '0.3878', 'learning_rate': '1.006e-05', 'ppl': '1.088', 'memory/max_active (GiB)': '65.22', 'memory/max_allocated (GiB)': '65.22', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '70.7', 'tokens/total': 3464128, 'tokens/trainable': 938304, 'epoch': '3.969'}

 99%|█████████▉| 127/128 [20:52<00:10, 10.33s/it]
100%|██████████| 128/128 [21:03<00:00, 10.34s/it]
                                                 
{'loss': '0.09788', 'grad_norm': '0.6063', 'learning_rate': '1.001e-05', 'ppl': '1.103', 'memory/max_active (GiB)': '62.03', 'memory/max_allocated (GiB)': '62.03', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '117.5', 'tokens/total': 3492372, 'tokens/trainable': 947116, 'epoch': '4'}

100%|██████████| 128/128 [21:03<00:00, 10.34s/it][2026-08-13 16:12:03,363] [INFO] [axolotl.core.trainers.base._save:828] [PID:1747] Saving model checkpoint to /workspace/python4-aft-v2-20260813T154138Z-control/experiments/python4/aft_v2/runs/20260813T154138Z/arms/control/train/checkpoints/checkpoint-128

                                                 
{'train_runtime': '1265', 'train_samples_per_second': '3.237', 'train_steps_per_second': '0.101', 'train_loss': '0.2132', 'memory/max_active (GiB)': '56.24', 'memory/max_allocated (GiB)': '56.24', 'memory/device_reserved (GiB)': '65.73', 'epoch': '4', 'tokens/train_per_sec_per_gpu': '0'}

100%|██████████| 128/128 [21:05<00:00, 10.34s/it]
100%|██████████| 128/128 [21:05<00:00,  9.88s/it]
[2026-08-13 16:12:05,324] [INFO] [axolotl.train.save_trained_model:267] [PID:1747] Training completed! Saving trained model to /workspace/python4-aft-v2-20260813T154138Z-control/experiments/python4/aft_v2/runs/20260813T154138Z/arms/control/train/checkpoints.
[2026-08-13 16:12:06,297] [INFO] [axolotl.train.save_trained_model:388] [PID:1747] Model successfully saved to /workspace/python4-aft-v2-20260813T154138Z-control/experiments/python4/aft_v2/runs/20260813T154138Z/arms/control/train/checkpoints