| [2026-08-13 15:48:49,616] [DEBUG] [axolotl.utils.config.log_gpu_memory_usage:127] [PID:1747] baseline 0.000GB () |
| [2026-08-13 15:48:49,617] [INFO] [axolotl.cli.config.load_cfg:333] [PID:1747] config: |
| { |
| "activation_offloading": false, |
| "adapter": "lora", |
| "attn_implementation": "flash_attention_2", |
| "attn_needs_dtype_cast": true, |
| "attn_supports_packing": true, |
| "attn_uses_flash_lib": true, |
| "axolotl_config_path": "/workspace/python4-aft-v2-20260813T154138Z-control/experiments/python4/aft_v2/runs/20260813T154138Z/arms/control/train/axolotl.yaml", |
| "base_model": "/workspace/python4-aft-v2-state/20260813T154138Z/control/parent/control/sft/end", |
| "base_model_config": "/workspace/python4-aft-v2-state/20260813T154138Z/control/parent/control/sft/end", |
| "batch_size": 32, |
| "bf16": true, |
| "capabilities": { |
| "bf16": true, |
| "compute_capability": "sm_90", |
| "fp8": true, |
| "n_gpu": 1, |
| "n_node": 1, |
| "tf32": true |
| }, |
| "chat_template": "gemma3", |
| "checkpoint_schedule": [ |
| 128 |
| ], |
| "context_parallel_size": 1, |
| "cosine_min_lr_ratio": 0.1, |
| "dataloader_num_workers": 1, |
| "dataloader_pin_memory": true, |
| "dataloader_prefetch_factor": 256, |
| "dataset_num_proc": 16, |
| "dataset_prepared_path": "/workspace/python4-aft-v2-20260813T154138Z-control/experiments/python4/aft_v2/runs/20260813T154138Z/arms/control/train/prepared", |
| "datasets": [ |
| { |
| "chat_template": "tokenizer_default", |
| "field_messages": "messages", |
| "message_property_mappings": { |
| "content": "content", |
| "role": "role" |
| }, |
| "path": "/workspace/python4-aft-v2-20260813T154138Z-control/experiments/python4/aft_v2/runs/20260813T154138Z/arms/control/aft_training.jsonl", |
| "trust_remote_code": false, |
| "type": "chat_template" |
| } |
| ], |
| "ddp": false, |
| "ddp_find_unused_parameters": true, |
| "device": "cuda:0", |
| "dion_rank_fraction": 1.0, |
| "dion_rank_multiple_of": 1, |
| "eaft_alpha": 1.0, |
| "eaft_k": 20, |
| "env_capabilities": { |
| "torch_version": "2.12.1" |
| }, |
| "eot_tokens": [ |
| "<end_of_turn>" |
| ], |
| "eval_batch_size": 4, |
| "eval_causal_lm_metrics": [ |
| "sacrebleu", |
| "comet", |
| "ter", |
| "chrf" |
| ], |
| "eval_max_new_tokens": 128, |
| "eval_table_size": 0, |
| "experimental_skip_move_to_device": true, |
| "fp16": false, |
| "generate_samples": false, |
| "generation_do_sample": true, |
| "generation_max_new_tokens": 50, |
| "generation_prompt_ratio": 0.5, |
| "generation_temperature": 0.7, |
| "gradient_accumulation_steps": 8, |
| "gradient_checkpointing": true, |
| "gradient_checkpointing_kwargs": { |
| "use_reentrant": false |
| }, |
| "include_tkps": true, |
| "is_multimodal": true, |
| "layer_offloading": false, |
| "learning_rate": 0.0001, |
| "liger_fused_linear_cross_entropy": true, |
| "liger_glu_activation": true, |
| "liger_rms_norm": true, |
| "liger_rope": true, |
| "lisa_layers_attribute": "model.layers", |
| "load_best_model_at_end": false, |
| "load_in_4bit": false, |
| "load_in_8bit": false, |
| "local_rank": 0, |
| "logging_steps": 1, |
| "lora_alpha": 128, |
| "lora_dropout": 0.0, |
| "lora_embedding_kernel": true, |
| "lora_mlp_kernel": true, |
| "lora_o_kernel": true, |
| "lora_qkv_kernel": true, |
| "lora_r": 64, |
| "lora_target_modules": [ |
| "model.language_model.layers.0.self_attn.q_proj", |
| "model.language_model.layers.0.self_attn.k_proj", |
| "model.language_model.layers.0.self_attn.v_proj", |
| "model.language_model.layers.0.self_attn.o_proj", |
| "model.language_model.layers.0.mlp.gate_proj", |
| "model.language_model.layers.0.mlp.up_proj", |
| "model.language_model.layers.0.mlp.down_proj", |
| "model.language_model.layers.1.self_attn.q_proj", |
| "model.language_model.layers.1.self_attn.k_proj", |
| "model.language_model.layers.1.self_attn.v_proj", |
| "model.language_model.layers.1.self_attn.o_proj", |
| "model.language_model.layers.1.mlp.gate_proj", |
| "model.language_model.layers.1.mlp.up_proj", |
| "model.language_model.layers.1.mlp.down_proj", |
| "model.language_model.layers.2.self_attn.q_proj", |
| "model.language_model.layers.2.self_attn.k_proj", |
| "model.language_model.layers.2.self_attn.v_proj", |
| "model.language_model.layers.2.self_attn.o_proj", |
| "model.language_model.layers.2.mlp.gate_proj", |
| "model.language_model.layers.2.mlp.up_proj", |
| "model.language_model.layers.2.mlp.down_proj", |
| "model.language_model.layers.3.self_attn.q_proj", |
| "model.language_model.layers.3.self_attn.k_proj", |
| "model.language_model.layers.3.self_attn.v_proj", |
| "model.language_model.layers.3.self_attn.o_proj", |
| "model.language_model.layers.3.mlp.gate_proj", |
| "model.language_model.layers.3.mlp.up_proj", |
| "model.language_model.layers.3.mlp.down_proj", |
| "model.language_model.layers.4.self_attn.q_proj", |
| "model.language_model.layers.4.self_attn.k_proj", |
| "model.language_model.layers.4.self_attn.v_proj", |
| "model.language_model.layers.4.self_attn.o_proj", |
| "model.language_model.layers.4.mlp.gate_proj", |
| "model.language_model.layers.4.mlp.up_proj", |
| "model.language_model.layers.4.mlp.down_proj", |
| "model.language_model.layers.5.self_attn.q_proj", |
| "model.language_model.layers.5.self_attn.k_proj", |
| "model.language_model.layers.5.self_attn.v_proj", |
| "model.language_model.layers.5.self_attn.o_proj", |
| "model.language_model.layers.5.mlp.gate_proj", |
| "model.language_model.layers.5.mlp.up_proj", |
| "model.language_model.layers.5.mlp.down_proj", |
| "model.language_model.layers.6.self_attn.q_proj", |
| "model.language_model.layers.6.self_attn.k_proj", |
| "model.language_model.layers.6.self_attn.v_proj", |
| "model.language_model.layers.6.self_attn.o_proj", |
| "model.language_model.layers.6.mlp.gate_proj", |
| "model.language_model.layers.6.mlp.up_proj", |
| "model.language_model.layers.6.mlp.down_proj", |
| "model.language_model.layers.7.self_attn.q_proj", |
| "model.language_model.layers.7.self_attn.k_proj", |
| "model.language_model.layers.7.self_attn.v_proj", |
| "model.language_model.layers.7.self_attn.o_proj", |
| "model.language_model.layers.7.mlp.gate_proj", |
| "model.language_model.layers.7.mlp.up_proj", |
| "model.language_model.layers.7.mlp.down_proj", |
| "model.language_model.layers.8.self_attn.q_proj", |
| "model.language_model.layers.8.self_attn.k_proj", |
| "model.language_model.layers.8.self_attn.v_proj", |
| "model.language_model.layers.8.self_attn.o_proj", |
| "model.language_model.layers.8.mlp.gate_proj", |
| "model.language_model.layers.8.mlp.up_proj", |
| "model.language_model.layers.8.mlp.down_proj", |
| "model.language_model.layers.9.self_attn.q_proj", |
| "model.language_model.layers.9.self_attn.k_proj", |
| "model.language_model.layers.9.self_attn.v_proj", |
| "model.language_model.layers.9.self_attn.o_proj", |
| "model.language_model.layers.9.mlp.gate_proj", |
| "model.language_model.layers.9.mlp.up_proj", |
| "model.language_model.layers.9.mlp.down_proj", |
| "model.language_model.layers.10.self_attn.q_proj", |
| "model.language_model.layers.10.self_attn.k_proj", |
| "model.language_model.layers.10.self_attn.v_proj", |
| "model.language_model.layers.10.self_attn.o_proj", |
| "model.language_model.layers.10.mlp.gate_proj", |
| "model.language_model.layers.10.mlp.up_proj", |
| "model.language_model.layers.10.mlp.down_proj", |
| "model.language_model.layers.11.self_attn.q_proj", |
| "model.language_model.layers.11.self_attn.k_proj", |
| "model.language_model.layers.11.self_attn.v_proj", |
| "model.language_model.layers.11.self_attn.o_proj", |
| "model.language_model.layers.11.mlp.gate_proj", |
| "model.language_model.layers.11.mlp.up_proj", |
| "model.language_model.layers.11.mlp.down_proj", |
| "model.language_model.layers.12.self_attn.q_proj", |
| "model.language_model.layers.12.self_attn.k_proj", |
| "model.language_model.layers.12.self_attn.v_proj", |
| "model.language_model.layers.12.self_attn.o_proj", |
| "model.language_model.layers.12.mlp.gate_proj", |
| "model.language_model.layers.12.mlp.up_proj", |
| "model.language_model.layers.12.mlp.down_proj", |
| "model.language_model.layers.13.self_attn.q_proj", |
| "model.language_model.layers.13.self_attn.k_proj", |
| "model.language_model.layers.13.self_attn.v_proj", |
| "model.language_model.layers.13.self_attn.o_proj", |
| "model.language_model.layers.13.mlp.gate_proj", |
| "model.language_model.layers.13.mlp.up_proj", |
| "model.language_model.layers.13.mlp.down_proj", |
| "model.language_model.layers.14.self_attn.q_proj", |
| "model.language_model.layers.14.self_attn.k_proj", |
| "model.language_model.layers.14.self_attn.v_proj", |
| "model.language_model.layers.14.self_attn.o_proj", |
| "model.language_model.layers.14.mlp.gate_proj", |
| "model.language_model.layers.14.mlp.up_proj", |
| "model.language_model.layers.14.mlp.down_proj", |
| "model.language_model.layers.15.self_attn.q_proj", |
| "model.language_model.layers.15.self_attn.k_proj", |
| "model.language_model.layers.15.self_attn.v_proj", |
| "model.language_model.layers.15.self_attn.o_proj", |
| "model.language_model.layers.15.mlp.gate_proj", |
| "model.language_model.layers.15.mlp.up_proj", |
| "model.language_model.layers.15.mlp.down_proj", |
| "model.language_model.layers.16.self_attn.q_proj", |
| "model.language_model.layers.16.self_attn.k_proj", |
| "model.language_model.layers.16.self_attn.v_proj", |
| "model.language_model.layers.16.self_attn.o_proj", |
| "model.language_model.layers.16.mlp.gate_proj", |
| "model.language_model.layers.16.mlp.up_proj", |
| "model.language_model.layers.16.mlp.down_proj", |
| "model.language_model.layers.17.self_attn.q_proj", |
| "model.language_model.layers.17.self_attn.k_proj", |
| "model.language_model.layers.17.self_attn.v_proj", |
| "model.language_model.layers.17.self_attn.o_proj", |
| "model.language_model.layers.17.mlp.gate_proj", |
| "model.language_model.layers.17.mlp.up_proj", |
| "model.language_model.layers.17.mlp.down_proj", |
| "model.language_model.layers.18.self_attn.q_proj", |
| "model.language_model.layers.18.self_attn.k_proj", |
| "model.language_model.layers.18.self_attn.v_proj", |
| "model.language_model.layers.18.self_attn.o_proj", |
| "model.language_model.layers.18.mlp.gate_proj", |
| "model.language_model.layers.18.mlp.up_proj", |
| "model.language_model.layers.18.mlp.down_proj", |
| "model.language_model.layers.19.self_attn.q_proj", |
| "model.language_model.layers.19.self_attn.k_proj", |
| "model.language_model.layers.19.self_attn.v_proj", |
| "model.language_model.layers.19.self_attn.o_proj", |
| "model.language_model.layers.19.mlp.gate_proj", |
| "model.language_model.layers.19.mlp.up_proj", |
| "model.language_model.layers.19.mlp.down_proj", |
| "model.language_model.layers.20.self_attn.q_proj", |
| "model.language_model.layers.20.self_attn.k_proj", |
| "model.language_model.layers.20.self_attn.v_proj", |
| "model.language_model.layers.20.self_attn.o_proj", |
| "model.language_model.layers.20.mlp.gate_proj", |
| "model.language_model.layers.20.mlp.up_proj", |
| "model.language_model.layers.20.mlp.down_proj", |
| "model.language_model.layers.21.self_attn.q_proj", |
| "model.language_model.layers.21.self_attn.k_proj", |
| "model.language_model.layers.21.self_attn.v_proj", |
| "model.language_model.layers.21.self_attn.o_proj", |
| "model.language_model.layers.21.mlp.gate_proj", |
| "model.language_model.layers.21.mlp.up_proj", |
| "model.language_model.layers.21.mlp.down_proj", |
| "model.language_model.layers.22.self_attn.q_proj", |
| "model.language_model.layers.22.self_attn.k_proj", |
| "model.language_model.layers.22.self_attn.v_proj", |
| "model.language_model.layers.22.self_attn.o_proj", |
| "model.language_model.layers.22.mlp.gate_proj", |
| "model.language_model.layers.22.mlp.up_proj", |
| "model.language_model.layers.22.mlp.down_proj", |
| "model.language_model.layers.23.self_attn.q_proj", |
| "model.language_model.layers.23.self_attn.k_proj", |
| "model.language_model.layers.23.self_attn.v_proj", |
| "model.language_model.layers.23.self_attn.o_proj", |
| "model.language_model.layers.23.mlp.gate_proj", |
| "model.language_model.layers.23.mlp.up_proj", |
| "model.language_model.layers.23.mlp.down_proj", |
| "model.language_model.layers.24.self_attn.q_proj", |
| "model.language_model.layers.24.self_attn.k_proj", |
| "model.language_model.layers.24.self_attn.v_proj", |
| "model.language_model.layers.24.self_attn.o_proj", |
| "model.language_model.layers.24.mlp.gate_proj", |
| "model.language_model.layers.24.mlp.up_proj", |
| "model.language_model.layers.24.mlp.down_proj", |
| "model.language_model.layers.25.self_attn.q_proj", |
| "model.language_model.layers.25.self_attn.k_proj", |
| "model.language_model.layers.25.self_attn.v_proj", |
| "model.language_model.layers.25.self_attn.o_proj", |
| "model.language_model.layers.25.mlp.gate_proj", |
| "model.language_model.layers.25.mlp.up_proj", |
| "model.language_model.layers.25.mlp.down_proj", |
| "model.language_model.layers.26.self_attn.q_proj", |
| "model.language_model.layers.26.self_attn.k_proj", |
| "model.language_model.layers.26.self_attn.v_proj", |
| "model.language_model.layers.26.self_attn.o_proj", |
| "model.language_model.layers.26.mlp.gate_proj", |
| "model.language_model.layers.26.mlp.up_proj", |
| "model.language_model.layers.26.mlp.down_proj", |
| "model.language_model.layers.27.self_attn.q_proj", |
| "model.language_model.layers.27.self_attn.k_proj", |
| "model.language_model.layers.27.self_attn.v_proj", |
| "model.language_model.layers.27.self_attn.o_proj", |
| "model.language_model.layers.27.mlp.gate_proj", |
| "model.language_model.layers.27.mlp.up_proj", |
| "model.language_model.layers.27.mlp.down_proj", |
| "model.language_model.layers.28.self_attn.q_proj", |
| "model.language_model.layers.28.self_attn.k_proj", |
| "model.language_model.layers.28.self_attn.v_proj", |
| "model.language_model.layers.28.self_attn.o_proj", |
| "model.language_model.layers.28.mlp.gate_proj", |
| "model.language_model.layers.28.mlp.up_proj", |
| "model.language_model.layers.28.mlp.down_proj", |
| "model.language_model.layers.29.self_attn.q_proj", |
| "model.language_model.layers.29.self_attn.k_proj", |
| "model.language_model.layers.29.self_attn.v_proj", |
| "model.language_model.layers.29.self_attn.o_proj", |
| "model.language_model.layers.29.mlp.gate_proj", |
| "model.language_model.layers.29.mlp.up_proj", |
| "model.language_model.layers.29.mlp.down_proj", |
| "model.language_model.layers.30.self_attn.q_proj", |
| "model.language_model.layers.30.self_attn.k_proj", |
| "model.language_model.layers.30.self_attn.v_proj", |
| "model.language_model.layers.30.self_attn.o_proj", |
| "model.language_model.layers.30.mlp.gate_proj", |
| "model.language_model.layers.30.mlp.up_proj", |
| "model.language_model.layers.30.mlp.down_proj", |
| "model.language_model.layers.31.self_attn.q_proj", |
| "model.language_model.layers.31.self_attn.k_proj", |
| "model.language_model.layers.31.self_attn.v_proj", |
| "model.language_model.layers.31.self_attn.o_proj", |
| "model.language_model.layers.31.mlp.gate_proj", |
| "model.language_model.layers.31.mlp.up_proj", |
| "model.language_model.layers.31.mlp.down_proj", |
| "model.language_model.layers.32.self_attn.q_proj", |
| "model.language_model.layers.32.self_attn.k_proj", |
| "model.language_model.layers.32.self_attn.v_proj", |
| "model.language_model.layers.32.self_attn.o_proj", |
| "model.language_model.layers.32.mlp.gate_proj", |
| "model.language_model.layers.32.mlp.up_proj", |
| "model.language_model.layers.32.mlp.down_proj", |
| "model.language_model.layers.33.self_attn.q_proj", |
| "model.language_model.layers.33.self_attn.k_proj", |
| "model.language_model.layers.33.self_attn.v_proj", |
| "model.language_model.layers.33.self_attn.o_proj", |
| "model.language_model.layers.33.mlp.gate_proj", |
| "model.language_model.layers.33.mlp.up_proj", |
| "model.language_model.layers.33.mlp.down_proj", |
| "model.language_model.layers.34.self_attn.q_proj", |
| "model.language_model.layers.34.self_attn.k_proj", |
| "model.language_model.layers.34.self_attn.v_proj", |
| "model.language_model.layers.34.self_attn.o_proj", |
| "model.language_model.layers.34.mlp.gate_proj", |
| "model.language_model.layers.34.mlp.up_proj", |
| "model.language_model.layers.34.mlp.down_proj", |
| "model.language_model.layers.35.self_attn.q_proj", |
| "model.language_model.layers.35.self_attn.k_proj", |
| "model.language_model.layers.35.self_attn.v_proj", |
| "model.language_model.layers.35.self_attn.o_proj", |
| "model.language_model.layers.35.mlp.gate_proj", |
| "model.language_model.layers.35.mlp.up_proj", |
| "model.language_model.layers.35.mlp.down_proj", |
| "model.language_model.layers.36.self_attn.q_proj", |
| "model.language_model.layers.36.self_attn.k_proj", |
| "model.language_model.layers.36.self_attn.v_proj", |
| "model.language_model.layers.36.self_attn.o_proj", |
| "model.language_model.layers.36.mlp.gate_proj", |
| "model.language_model.layers.36.mlp.up_proj", |
| "model.language_model.layers.36.mlp.down_proj", |
| "model.language_model.layers.37.self_attn.q_proj", |
| "model.language_model.layers.37.self_attn.k_proj", |
| "model.language_model.layers.37.self_attn.v_proj", |
| "model.language_model.layers.37.self_attn.o_proj", |
| "model.language_model.layers.37.mlp.gate_proj", |
| "model.language_model.layers.37.mlp.up_proj", |
| "model.language_model.layers.37.mlp.down_proj", |
| "model.language_model.layers.38.self_attn.q_proj", |
| "model.language_model.layers.38.self_attn.k_proj", |
| "model.language_model.layers.38.self_attn.v_proj", |
| "model.language_model.layers.38.self_attn.o_proj", |
| "model.language_model.layers.38.mlp.gate_proj", |
| "model.language_model.layers.38.mlp.up_proj", |
| "model.language_model.layers.38.mlp.down_proj", |
| "model.language_model.layers.39.self_attn.q_proj", |
| "model.language_model.layers.39.self_attn.k_proj", |
| "model.language_model.layers.39.self_attn.v_proj", |
| "model.language_model.layers.39.self_attn.o_proj", |
| "model.language_model.layers.39.mlp.gate_proj", |
| "model.language_model.layers.39.mlp.up_proj", |
| "model.language_model.layers.39.mlp.down_proj", |
| "model.language_model.layers.40.self_attn.q_proj", |
| "model.language_model.layers.40.self_attn.k_proj", |
| "model.language_model.layers.40.self_attn.v_proj", |
| "model.language_model.layers.40.self_attn.o_proj", |
| "model.language_model.layers.40.mlp.gate_proj", |
| "model.language_model.layers.40.mlp.up_proj", |
| "model.language_model.layers.40.mlp.down_proj", |
| "model.language_model.layers.41.self_attn.q_proj", |
| "model.language_model.layers.41.self_attn.k_proj", |
| "model.language_model.layers.41.self_attn.v_proj", |
| "model.language_model.layers.41.self_attn.o_proj", |
| "model.language_model.layers.41.mlp.gate_proj", |
| "model.language_model.layers.41.mlp.up_proj", |
| "model.language_model.layers.41.mlp.down_proj", |
| "model.language_model.layers.42.self_attn.q_proj", |
| "model.language_model.layers.42.self_attn.k_proj", |
| "model.language_model.layers.42.self_attn.v_proj", |
| "model.language_model.layers.42.self_attn.o_proj", |
| "model.language_model.layers.42.mlp.gate_proj", |
| "model.language_model.layers.42.mlp.up_proj", |
| "model.language_model.layers.42.mlp.down_proj", |
| "model.language_model.layers.43.self_attn.q_proj", |
| "model.language_model.layers.43.self_attn.k_proj", |
| "model.language_model.layers.43.self_attn.v_proj", |
| "model.language_model.layers.43.self_attn.o_proj", |
| "model.language_model.layers.43.mlp.gate_proj", |
| "model.language_model.layers.43.mlp.up_proj", |
| "model.language_model.layers.43.mlp.down_proj", |
| "model.language_model.layers.44.self_attn.q_proj", |
| "model.language_model.layers.44.self_attn.k_proj", |
| "model.language_model.layers.44.self_attn.v_proj", |
| "model.language_model.layers.44.self_attn.o_proj", |
| "model.language_model.layers.44.mlp.gate_proj", |
| "model.language_model.layers.44.mlp.up_proj", |
| "model.language_model.layers.44.mlp.down_proj", |
| "model.language_model.layers.45.self_attn.q_proj", |
| "model.language_model.layers.45.self_attn.k_proj", |
| "model.language_model.layers.45.self_attn.v_proj", |
| "model.language_model.layers.45.self_attn.o_proj", |
| "model.language_model.layers.45.mlp.gate_proj", |
| "model.language_model.layers.45.mlp.up_proj", |
| "model.language_model.layers.45.mlp.down_proj", |
| "model.language_model.layers.46.self_attn.q_proj", |
| "model.language_model.layers.46.self_attn.k_proj", |
| "model.language_model.layers.46.self_attn.v_proj", |
| "model.language_model.layers.46.self_attn.o_proj", |
| "model.language_model.layers.46.mlp.gate_proj", |
| "model.language_model.layers.46.mlp.up_proj", |
| "model.language_model.layers.46.mlp.down_proj", |
| "model.language_model.layers.47.self_attn.q_proj", |
| "model.language_model.layers.47.self_attn.k_proj", |
| "model.language_model.layers.47.self_attn.v_proj", |
| "model.language_model.layers.47.self_attn.o_proj", |
| "model.language_model.layers.47.mlp.gate_proj", |
| "model.language_model.layers.47.mlp.up_proj", |
| "model.language_model.layers.47.mlp.down_proj", |
| "model.language_model.layers.48.self_attn.q_proj", |
| "model.language_model.layers.48.self_attn.k_proj", |
| "model.language_model.layers.48.self_attn.v_proj", |
| "model.language_model.layers.48.self_attn.o_proj", |
| "model.language_model.layers.48.mlp.gate_proj", |
| "model.language_model.layers.48.mlp.up_proj", |
| "model.language_model.layers.48.mlp.down_proj", |
| "model.language_model.layers.49.self_attn.q_proj", |
| "model.language_model.layers.49.self_attn.k_proj", |
| "model.language_model.layers.49.self_attn.v_proj", |
| "model.language_model.layers.49.self_attn.o_proj", |
| "model.language_model.layers.49.mlp.gate_proj", |
| "model.language_model.layers.49.mlp.up_proj", |
| "model.language_model.layers.49.mlp.down_proj", |
| "model.language_model.layers.50.self_attn.q_proj", |
| "model.language_model.layers.50.self_attn.k_proj", |
| "model.language_model.layers.50.self_attn.v_proj", |
| "model.language_model.layers.50.self_attn.o_proj", |
| "model.language_model.layers.50.mlp.gate_proj", |
| "model.language_model.layers.50.mlp.up_proj", |
| "model.language_model.layers.50.mlp.down_proj", |
| "model.language_model.layers.51.self_attn.q_proj", |
| "model.language_model.layers.51.self_attn.k_proj", |
| "model.language_model.layers.51.self_attn.v_proj", |
| "model.language_model.layers.51.self_attn.o_proj", |
| "model.language_model.layers.51.mlp.gate_proj", |
| "model.language_model.layers.51.mlp.up_proj", |
| "model.language_model.layers.51.mlp.down_proj", |
| "model.language_model.layers.52.self_attn.q_proj", |
| "model.language_model.layers.52.self_attn.k_proj", |
| "model.language_model.layers.52.self_attn.v_proj", |
| "model.language_model.layers.52.self_attn.o_proj", |
| "model.language_model.layers.52.mlp.gate_proj", |
| "model.language_model.layers.52.mlp.up_proj", |
| "model.language_model.layers.52.mlp.down_proj", |
| "model.language_model.layers.53.self_attn.q_proj", |
| "model.language_model.layers.53.self_attn.k_proj", |
| "model.language_model.layers.53.self_attn.v_proj", |
| "model.language_model.layers.53.self_attn.o_proj", |
| "model.language_model.layers.53.mlp.gate_proj", |
| "model.language_model.layers.53.mlp.up_proj", |
| "model.language_model.layers.53.mlp.down_proj", |
| "model.language_model.layers.54.self_attn.q_proj", |
| "model.language_model.layers.54.self_attn.k_proj", |
| "model.language_model.layers.54.self_attn.v_proj", |
| "model.language_model.layers.54.self_attn.o_proj", |
| "model.language_model.layers.54.mlp.gate_proj", |
| "model.language_model.layers.54.mlp.up_proj", |
| "model.language_model.layers.54.mlp.down_proj", |
| "model.language_model.layers.55.self_attn.q_proj", |
| "model.language_model.layers.55.self_attn.k_proj", |
| "model.language_model.layers.55.self_attn.v_proj", |
| "model.language_model.layers.55.self_attn.o_proj", |
| "model.language_model.layers.55.mlp.gate_proj", |
| "model.language_model.layers.55.mlp.up_proj", |
| "model.language_model.layers.55.mlp.down_proj", |
| "model.language_model.layers.56.self_attn.q_proj", |
| "model.language_model.layers.56.self_attn.k_proj", |
| "model.language_model.layers.56.self_attn.v_proj", |
| "model.language_model.layers.56.self_attn.o_proj", |
| "model.language_model.layers.56.mlp.gate_proj", |
| "model.language_model.layers.56.mlp.up_proj", |
| "model.language_model.layers.56.mlp.down_proj", |
| "model.language_model.layers.57.self_attn.q_proj", |
| "model.language_model.layers.57.self_attn.k_proj", |
| "model.language_model.layers.57.self_attn.v_proj", |
| "model.language_model.layers.57.self_attn.o_proj", |
| "model.language_model.layers.57.mlp.gate_proj", |
| "model.language_model.layers.57.mlp.up_proj", |
| "model.language_model.layers.57.mlp.down_proj", |
| "model.language_model.layers.58.self_attn.q_proj", |
| "model.language_model.layers.58.self_attn.k_proj", |
| "model.language_model.layers.58.self_attn.v_proj", |
| "model.language_model.layers.58.self_attn.o_proj", |
| "model.language_model.layers.58.mlp.gate_proj", |
| "model.language_model.layers.58.mlp.up_proj", |
| "model.language_model.layers.58.mlp.down_proj", |
| "model.language_model.layers.59.self_attn.q_proj", |
| "model.language_model.layers.59.self_attn.k_proj", |
| "model.language_model.layers.59.self_attn.v_proj", |
| "model.language_model.layers.59.self_attn.o_proj", |
| "model.language_model.layers.59.mlp.gate_proj", |
| "model.language_model.layers.59.mlp.up_proj", |
| "model.language_model.layers.59.mlp.down_proj", |
| "model.language_model.layers.60.self_attn.q_proj", |
| "model.language_model.layers.60.self_attn.k_proj", |
| "model.language_model.layers.60.self_attn.v_proj", |
| "model.language_model.layers.60.self_attn.o_proj", |
| "model.language_model.layers.60.mlp.gate_proj", |
| "model.language_model.layers.60.mlp.up_proj", |
| "model.language_model.layers.60.mlp.down_proj", |
| "model.language_model.layers.61.self_attn.q_proj", |
| "model.language_model.layers.61.self_attn.k_proj", |
| "model.language_model.layers.61.self_attn.v_proj", |
| "model.language_model.layers.61.self_attn.o_proj", |
| "model.language_model.layers.61.mlp.gate_proj", |
| "model.language_model.layers.61.mlp.up_proj", |
| "model.language_model.layers.61.mlp.down_proj" |
| ], |
| "loraplus_lr_embedding": 1e-06, |
| "lr_scheduler": "cosine", |
| "max_grad_norm": 1.0, |
| "mean_resizing_embeddings": false, |
| "merge_method": "memory_efficient", |
| "micro_batch_size": 4, |
| "model_config_type": "gemma3", |
| "model_config_type_text": "gemma3_text", |
| "num_epochs": 4.0, |
| "num_generation_samples": 3, |
| "optimizer": "adamw_torch_fused", |
| "otel_metrics_host": "localhost", |
| "otel_metrics_port": 8000, |
| "output_dir": "/workspace/python4-aft-v2-20260813T154138Z-control/experiments/python4/aft_v2/runs/20260813T154138Z/arms/control/train/checkpoints", |
| "pad_to_sequence_len": false, |
| "plugins": [ |
| "axolotl.integrations.liger.LigerPlugin", |
| "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" |
| ], |
| "pretrain_multipack_attn": true, |
| "processor_config": "/workspace/python4-aft-v2-state/20260813T154138Z/control/parent/control/sft/end", |
| "profiler_steps_start": 0, |
| "qgalore_cos_threshold": 0.4, |
| "qgalore_gamma_proj": 2, |
| "qgalore_proj_bits": 4, |
| "qgalore_proj_group_size": 256, |
| "qgalore_proj_quant": true, |
| "qgalore_proj_type": "std", |
| "qgalore_queue_size": 5, |
| "qgalore_rank": 256, |
| "qgalore_scale": 0.25, |
| "qgalore_update_proj_gap": 200, |
| "qlora_sharded_model_loading": false, |
| "quantize_moe_experts": false, |
| "ray_num_workers": 1, |
| "relora_prune_method": "magnitude", |
| "resources_per_worker": { |
| "GPU": 1 |
| }, |
| "sample_packing": false, |
| "sample_packing_bin_size": 200, |
| "sample_packing_group_size": 100000, |
| "save_only_model": true, |
| "save_safetensors": true, |
| "save_strategy": "no", |
| "save_total_limit": 1, |
| "seed": 424242, |
| "sequence_len": 4096, |
| "shuffle_before_merging_datasets": false, |
| "shuffle_merged_datasets": true, |
| "skip_prepare_dataset": false, |
| "streaming_multipack_buffer_size": 10000, |
| "strict": false, |
| "tensor_parallel_size": 1, |
| "tf32": true, |
| "tiled_mlp_use_original_mlp": true, |
| "tokenizer_config": "/workspace/python4-aft-v2-state/20260813T154138Z/control/parent/control/sft/end", |
| "tokenizer_save_jinja_files": true, |
| "torch_dtype": "torch.bfloat16", |
| "train_on_inputs": false, |
| "trl": { |
| "async_prefetch": false, |
| "log_completions": false, |
| "mask_truncated_completions": false, |
| "ref_model_mixup_alpha": 0.9, |
| "ref_model_sync_steps": 64, |
| "replay_buffer_size": 0, |
| "replay_recompute_logps": true, |
| "reroll_max_groups": 1, |
| "reroll_start_fraction": 1.0, |
| "reward_num_workers": 1, |
| "scale_rewards": true, |
| "skip_zero_advantage_batches": true, |
| "sync_ref_model": false, |
| "use_data_producer": false, |
| "use_vllm": false, |
| "vllm_lora_sync": false, |
| "vllm_server_host": "0.0.0.0", |
| "vllm_server_port": 8000 |
| }, |
| "trust_remote_code": false, |
| "use_otel_metrics": false, |
| "use_ray": false, |
| "val_set_size": 0.0, |
| "vllm": { |
| "device": "auto", |
| "dtype": "auto", |
| "gpu_memory_utilization": 0.9, |
| "host": "0.0.0.0", |
| "port": 8000 |
| }, |
| "warmup_ratio": 0.05, |
| "weight_decay": 0.01, |
| "world_size": 1 |
| } |
| [2026-08-13 15:48:50,617] [DEBUG] [axolotl.loaders.utils.check_model_config:88] [PID:1747] Loaded image size: 896 from model config |
| [2026-08-13 15:48:52,258] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:311] [PID:1747] EOS: 106 / <end_of_turn> |
| [2026-08-13 15:48:52,258] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:312] [PID:1747] BOS: 2 / <bos> |
| [2026-08-13 15:48:52,258] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:313] [PID:1747] PAD: 0 / <pad> |
| [2026-08-13 15:48:52,258] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:314] [PID:1747] UNK: 3 / <unk> |
| [2026-08-13 15:48:52,259] [INFO] [axolotl.utils.data.shared.load_preprocessed_dataset:482] [PID:1747] Unable to find prepared dataset in /workspace/python4-aft-v2-20260813T154138Z-control/experiments/python4/aft_v2/runs/20260813T154138Z/arms/control/train/prepared/cadb634638434a88a5119b219d5e7b61 |
| [2026-08-13 15:48:52,259] [INFO] [axolotl.utils.data.sft._load_raw_datasets:320] [PID:1747] Loading raw datasets... |
| [2026-08-13 15:48:52,259] [WARNING] [axolotl.utils.data.sft._load_raw_datasets:322] [PID:1747] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. |
|
Generating train split: 0 examples [00:00, ? examples/s]
Generating train split: 1024 examples [00:00, 171750.60 examples/s] |
| [2026-08-13 15:48:52,729] [INFO] [axolotl.utils.data.wrappers.get_dataset_wrapper:87] [PID:1747] Loading dataset: /workspace/python4-aft-v2-20260813T154138Z-control/experiments/python4/aft_v2/runs/20260813T154138Z/arms/control/aft_training.jsonl with base_type: chat_template and prompt_style: None |
| [2026-08-13 15:48:52,736] [INFO] [axolotl.prompt_strategies.chat_template.__call__:1209] [PID:1747] Using chat template: |
| --- |
| {{ bos_token }} |
| {%- if messages[0]['role'] == 'system' -%} |
| {%- if messages[0]['content'] is string -%} |
| {%- set first_user_prefix = messages[0]['content'] + ' |
| |
| ' -%} |
| {%- else -%} |
| {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' |
| |
| ' -%} |
| {%- endif -%} |
| {%- set loop_messages = messages[1:] -%} |
| {%- else -%} |
| {%- set first_user_prefix = "" -%} |
| {%- set loop_messages = messages -%} |
| {%- endif -%} |
| {%- for message in loop_messages -%} |
| {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} |
| {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} |
| {%- endif -%} |
| {%- if (message['role'] == 'assistant') -%} |
| {%- set role = "model" -%} |
| {%- else -%} |
| {%- set role = message['role'] -%} |
| {%- endif -%} |
| {{ '<start_of_turn>' + role + ' |
| ' + (first_user_prefix if loop.first else "") }} |
| {%- if message['content'] is string -%} |
| {{ message['content'] | trim }} |
| {%- elif message['content'] is iterable -%} |
| {%- for item in message['content'] -%} |
| {%- if item['type'] == 'image' -%} |
| {{ '<start_of_image>' }} |
| {%- elif item['type'] == 'text' -%} |
| {{ item['text'] | trim }} |
| {%- endif -%} |
| {%- endfor -%} |
| {%- else -%} |
| {{ raise_exception("Invalid content type") }} |
| {%- endif -%} |
| {{ '<end_of_turn> |
| ' }} |
| {%- endfor -%} |
| {%- if add_generation_prompt -%} |
| {{'<start_of_turn>model |
| '}} |
| {%- endif -%} |
|
|
| --- |
|
Tokenizing Prompts (num_proc=16): 0%| | 0/1024 [00:00<?, ? examples/s]
Tokenizing Prompts (num_proc=16): 6%|▋ | 64/1024 [00:05<01:28, 10.89 examples/s]
Tokenizing Prompts (num_proc=16): 12%|█▎ | 128/1024 [00:09<01:04, 13.82 examples/s]
Tokenizing Prompts (num_proc=16): 19%|█▉ | 192/1024 [00:13<00:56, 14.84 examples/s]
Tokenizing Prompts (num_proc=16): 25%|██▌ | 256/1024 [00:17<00:51, 15.06 examples/s]
Tokenizing Prompts (num_proc=16): 31%|███▏ | 320/1024 [00:21<00:45, 15.47 examples/s]
Tokenizing Prompts (num_proc=16): 38%|███▊ | 384/1024 [00:25<00:40, 15.84 examples/s]
Tokenizing Prompts (num_proc=16): 44%|████▍ | 448/1024 [00:29<00:35, 16.04 examples/s]
Tokenizing Prompts (num_proc=16): 50%|█████ | 512/1024 [00:33<00:31, 16.17 examples/s]
Tokenizing Prompts (num_proc=16): 56%|█████▋ | 576/1024 [00:37<00:27, 16.14 examples/s]
Tokenizing Prompts (num_proc=16): 62%|██████▎ | 640/1024 [00:41<00:23, 16.19 examples/s]
Tokenizing Prompts (num_proc=16): 69%|██████▉ | 704/1024 [00:45<00:19, 16.20 examples/s]
Tokenizing Prompts (num_proc=16): 75%|███████▌ | 768/1024 [00:48<00:15, 16.37 examples/s]
Tokenizing Prompts (num_proc=16): 81%|████████▏ | 832/1024 [00:52<00:11, 16.28 examples/s]
Tokenizing Prompts (num_proc=16): 88%|████████▊ | 896/1024 [00:56<00:07, 16.37 examples/s]
Tokenizing Prompts (num_proc=16): 94%|█████████▍| 960/1024 [01:00<00:03, 16.06 examples/s]
Tokenizing Prompts (num_proc=16): 100%|██████████| 1024/1024 [01:04<00:00, 16.32 examples/s]
Tokenizing Prompts (num_proc=16): 100%|██████████| 1024/1024 [01:05<00:00, 15.74 examples/s] |
| [2026-08-13 15:50:27,061] [INFO] [axolotl.utils.data.utils._log_dataset_stats:212] [PID:1747] min_input_len: 255 |
| [2026-08-13 15:50:27,061] [INFO] [axolotl.utils.data.utils._log_dataset_stats:213] [PID:1747] max_input_len: 2091 |
|
Dropping Invalid Sequences (<None or >4096) (num_proc=16): 0%| | 0/1024 [00:00<?, ? examples/s]
Dropping Invalid Sequences (<None or >4096) (num_proc=16): 6%|▋ | 64/1024 [00:00<00:05, 189.02 examples/s]
Dropping Invalid Sequences (<None or >4096) (num_proc=16): 100%|██████████| 1024/1024 [00:00<00:00, 1687.50 examples/s] |
|
Saving the dataset (0/4 shards): 0%| | 0/1024 [00:00<?, ? examples/s]
Saving the dataset (0/4 shards): 25%|██▌ | 256/1024 [00:06<00:20, 37.79 examples/s]
Saving the dataset (1/4 shards): 25%|██▌ | 256/1024 [00:06<00:20, 37.79 examples/s]
Saving the dataset (2/4 shards): 50%|█████ | 512/1024 [00:06<00:13, 37.79 examples/s]
Saving the dataset (3/4 shards): 75%|███████▌ | 768/1024 [00:06<00:06, 37.79 examples/s]
Saving the dataset (4/4 shards): 100%|██████████| 1024/1024 [00:06<00:00, 37.79 examples/s]
Saving the dataset (4/4 shards): 100%|██████████| 1024/1024 [00:07<00:00, 129.45 examples/s] |
| [2026-08-13 15:50:35,788] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:420] [PID:1747] total_num_tokens: 654_917 |
| [2026-08-13 15:50:35,802] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:438] [PID:1747] `total_supervised_tokens: 236_779` |
| [2026-08-13 15:50:35,802] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:521] [PID:1747] total_num_steps: 128 |
| [2026-08-13 15:50:35,802] [INFO] [axolotl.utils.data.sft._prepare_standard_dataset:121] [PID:1747] Maximum number of steps set at 128 |
| [2026-08-13 15:50:35,803] [DEBUG] [axolotl.train.setup_model_and_tokenizer:70] [PID:1747] loading tokenizer... /workspace/python4-aft-v2-state/20260813T154138Z/control/parent/control/sft/end |
| [2026-08-13 15:50:37,630] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:311] [PID:1747] EOS: 106 / <end_of_turn> |
| [2026-08-13 15:50:37,630] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:312] [PID:1747] BOS: 2 / <bos> |
| [2026-08-13 15:50:37,630] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:313] [PID:1747] PAD: 0 / <pad> |
| [2026-08-13 15:50:37,630] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:314] [PID:1747] UNK: 3 / <unk> |
| [2026-08-13 15:50:39,540] [DEBUG] [axolotl.train.setup_model_and_tokenizer:81] [PID:1747] Loading model |
| [2026-08-13 15:50:39,545] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:75] [PID:1747] Patched OptimState8bit for torch.compile compatibility |
| [2026-08-13 15:50:39,545] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:122] [PID:1747] Patched OptimState4bit for torch.compile compatibility |
| [2026-08-13 15:50:39,545] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:154] [PID:1747] Patched OptimStateFp8 for torch.compile compatibility |
| [2026-08-13 15:50:39,550] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_evaluation_loop:94] [PID:1747] Patched Trainer.evaluation_loop with nanmean loss calculation |
| [2026-08-13 15:50:39,551] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_maybe_log_save_evaluate:148] [PID:1747] Patched Trainer._maybe_log_save_evaluate with nanmean loss calculation |
| [2026-08-13 15:50:39,621] [INFO] [axolotl.monkeypatch.attention.flash_attn_4.patch_flash_attn_4:52] [PID:1747] Flash Attention 4 is available for your GPU and offers faster training speeds. To enable: pip install flash-attn-4 |
| [2026-08-13 15:50:40,840] [INFO] [axolotl.monkeypatch.lora_kernels.patch_self_attn_lora:304] [PID:1747] Patched attention class with LoRA optims: Gemma3Attention |
| [2026-08-13 15:50:41,076] [INFO] [axolotl.integrations.liger.plugin.pre_model_load:117] [PID:1747] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} |
|
Loading weights: 0%| | 0/1248 [00:00<?, ?it/s]
Loading weights: 41%|████▏ | 516/1248 [00:00<00:00, 5149.35it/s]
Loading weights: 83%|████████▎ | 1031/1248 [00:00<00:00, 4747.31it/s]
Loading weights: 100%|██████████| 1248/1248 [00:00<00:00, 4958.27it/s] |
| [2026-08-13 15:50:45,860] [INFO] [axolotl.loaders.model._configure_embedding_dtypes:433] [PID:1747] Converting modules to torch.bfloat16 |
| [2026-08-13 15:50:47,761] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:1747] Memory usage after model load 0.000GB () |
| trainable params: 454,066,176 || all params: 27,886,472,816 || trainable%: 1.6283 |
| [2026-08-13 15:50:50,831] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:1747] after adapters 0.000GB () |
| [2026-08-13 15:50:58,971] [INFO] [axolotl.train.save_initial_configs:450] [PID:1747] Pre-saving adapter config to /workspace/python4-aft-v2-20260813T154138Z-control/experiments/python4/aft_v2/runs/20260813T154138Z/arms/control/train/checkpoints... |
| [2026-08-13 15:50:58,972] [INFO] [axolotl.train.save_initial_configs:454] [PID:1747] Pre-saving tokenizer to /workspace/python4-aft-v2-20260813T154138Z-control/experiments/python4/aft_v2/runs/20260813T154138Z/arms/control/train/checkpoints... |
| [2026-08-13 15:50:59,306] [INFO] [axolotl.train.save_initial_configs:459] [PID:1747] Pre-saving model config to /workspace/python4-aft-v2-20260813T154138Z-control/experiments/python4/aft_v2/runs/20260813T154138Z/arms/control/train/checkpoints... |
| [2026-08-13 15:50:59,309] [INFO] [axolotl.train.save_initial_configs:463] [PID:1747] Pre-saving processor to /workspace/python4-aft-v2-20260813T154138Z-control/experiments/python4/aft_v2/runs/20260813T154138Z/arms/control/train/checkpoints... |
| [2026-08-13 15:50:59,602] [INFO] [axolotl.train.execute_training:226] [PID:1747] Starting trainer... |
|
0%| | 0/128 [00:00<?, ?it/s]
1%| | 1/128 [00:13<28:31, 13.48s/it]
{'loss': '0.8531', 'grad_norm': '1.081', 'learning_rate': '0', 'ppl': '2.347', 'memory/max_active (GiB)': '59.46', 'memory/max_allocated (GiB)': '59.46', 'memory/device_reserved (GiB)': '60.14', 'tokens/train_per_sec_per_gpu': '109.6', 'tokens/total': 28572, 'tokens/trainable': 7575, 'epoch': '0.03125'} |
|
1%| | 1/128 [00:13<28:31, 13.48s/it]
2%|▏ | 2/128 [00:23<24:25, 11.63s/it]
{'loss': '0.955', 'grad_norm': '0.9706', 'learning_rate': '1.667e-05', 'ppl': '2.599', 'memory/max_active (GiB)': '61.77', 'memory/max_allocated (GiB)': '61.77', 'memory/device_reserved (GiB)': '62.02', 'tokens/train_per_sec_per_gpu': '120.5', 'tokens/total': 56512, 'tokens/trainable': 15865, 'epoch': '0.0625'} |
|
2%|▏ | 2/128 [00:23<24:25, 11.63s/it]
2%|▏ | 3/128 [00:33<22:00, 10.56s/it]
{'loss': '0.8412', 'grad_norm': '1.071', 'learning_rate': '3.333e-05', 'ppl': '2.319', 'memory/max_active (GiB)': '61.94', 'memory/max_allocated (GiB)': '61.94', 'memory/device_reserved (GiB)': '62.28', 'tokens/train_per_sec_per_gpu': '83.8', 'tokens/total': 82320, 'tokens/trainable': 23211, 'epoch': '0.09375'} |
|
2%|▏ | 3/128 [00:33<22:00, 10.56s/it]
3%|▎ | 4/128 [00:43<22:00, 10.65s/it]
{'loss': '0.7129', 'grad_norm': '0.9637', 'learning_rate': '5e-05', 'ppl': '2.04', 'memory/max_active (GiB)': '62.39', 'memory/max_allocated (GiB)': '62.39', 'memory/device_reserved (GiB)': '62.74', 'tokens/train_per_sec_per_gpu': '85.97', 'tokens/total': 112500, 'tokens/trainable': 30046, 'epoch': '0.125'} |
|
3%|▎ | 4/128 [00:43<22:00, 10.65s/it]
4%|▍ | 5/128 [00:54<21:35, 10.53s/it]
{'loss': '0.5368', 'grad_norm': '0.7934', 'learning_rate': '6.667e-05', 'ppl': '1.711', 'memory/max_active (GiB)': '65.24', 'memory/max_allocated (GiB)': '65.24', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '105.2', 'tokens/total': 142096, 'tokens/trainable': 37315, 'epoch': '0.1562'} |
|
4%|▍ | 5/128 [00:54<21:35, 10.53s/it]
5%|▍ | 6/128 [01:03<20:52, 10.26s/it]
{'loss': '0.5395', 'grad_norm': '0.9188', 'learning_rate': '8.333e-05', 'ppl': '1.715', 'memory/max_active (GiB)': '62.16', 'memory/max_allocated (GiB)': '62.16', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '101.9', 'tokens/total': 168284, 'tokens/trainable': 44600, 'epoch': '0.1875'} |
|
5%|▍ | 6/128 [01:03<20:52, 10.26s/it]
5%|▌ | 7/128 [01:13<20:32, 10.19s/it]
{'loss': '0.4757', 'grad_norm': '0.7304', 'learning_rate': '0.0001', 'ppl': '1.609', 'memory/max_active (GiB)': '61.52', 'memory/max_allocated (GiB)': '61.52', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '115.7', 'tokens/total': 196820, 'tokens/trainable': 52174, 'epoch': '0.2188'} |
|
5%|▌ | 7/128 [01:13<20:32, 10.19s/it]
6%|▋ | 8/128 [01:24<20:33, 10.28s/it]
{'loss': '0.3853', 'grad_norm': '0.6223', 'learning_rate': '9.999e-05', 'ppl': '1.47', 'memory/max_active (GiB)': '62.62', 'memory/max_allocated (GiB)': '62.62', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '116.3', 'tokens/total': 226052, 'tokens/trainable': 59475, 'epoch': '0.25'} |
|
6%|▋ | 8/128 [01:24<20:33, 10.28s/it]
7%|▋ | 9/128 [01:33<19:41, 9.93s/it]
{'loss': '0.3623', 'grad_norm': '0.4904', 'learning_rate': '9.994e-05', 'ppl': '1.437', 'memory/max_active (GiB)': '62.4', 'memory/max_allocated (GiB)': '62.4', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '72.07', 'tokens/total': 251360, 'tokens/trainable': 66221, 'epoch': '0.2812'} |
|
7%|▋ | 9/128 [01:33<19:41, 9.93s/it]
8%|▊ | 10/128 [01:44<20:07, 10.23s/it]
{'loss': '0.3182', 'grad_norm': '0.4618', 'learning_rate': '9.987e-05', 'ppl': '1.375', 'memory/max_active (GiB)': '62.99', 'memory/max_allocated (GiB)': '62.99', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '105.7', 'tokens/total': 282048, 'tokens/trainable': 74909, 'epoch': '0.3125'} |
|
8%|▊ | 10/128 [01:44<20:07, 10.23s/it]
9%|▊ | 11/128 [01:53<18:54, 9.70s/it]
{'loss': '0.2723', 'grad_norm': '0.4428', 'learning_rate': '9.976e-05', 'ppl': '1.313', 'memory/max_active (GiB)': '61.38', 'memory/max_allocated (GiB)': '61.38', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '67.89', 'tokens/total': 304988, 'tokens/trainable': 80926, 'epoch': '0.3438'} |
|
9%|▊ | 11/128 [01:53<18:54, 9.70s/it]
9%|▉ | 12/128 [02:03<19:02, 9.85s/it]
{'loss': '0.397', 'grad_norm': '0.4366', 'learning_rate': '9.963e-05', 'ppl': '1.487', 'memory/max_active (GiB)': '61.88', 'memory/max_allocated (GiB)': '61.88', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '82.91', 'tokens/total': 333108, 'tokens/trainable': 87992, 'epoch': '0.375'} |
|
9%|▉ | 12/128 [02:03<19:02, 9.85s/it]
10%|█ | 13/128 [02:12<18:25, 9.62s/it]
{'loss': '0.6852', 'grad_norm': '0.4224', 'learning_rate': '9.946e-05', 'ppl': '1.984', 'memory/max_active (GiB)': '60.88', 'memory/max_allocated (GiB)': '60.88', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '150.9', 'tokens/total': 356952, 'tokens/trainable': 96195, 'epoch': '0.4062'} |
|
10%|█ | 13/128 [02:12<18:25, 9.62s/it]
11%|█ | 14/128 [02:21<18:17, 9.62s/it]
{'loss': '0.3385', 'grad_norm': '0.5395', 'learning_rate': '9.927e-05', 'ppl': '1.403', 'memory/max_active (GiB)': '61.4', 'memory/max_allocated (GiB)': '61.4', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '67.64', 'tokens/total': 384392, 'tokens/trainable': 103298, 'epoch': '0.4375'} |
|
11%|█ | 14/128 [02:21<18:17, 9.62s/it]
12%|█▏ | 15/128 [02:32<18:33, 9.85s/it]
{'loss': '0.3099', 'grad_norm': '0.4232', 'learning_rate': '9.905e-05', 'ppl': '1.363', 'memory/max_active (GiB)': '62.24', 'memory/max_allocated (GiB)': '62.24', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '61.44', 'tokens/total': 413280, 'tokens/trainable': 109986, 'epoch': '0.4688'} |
|
12%|█▏ | 15/128 [02:32<18:33, 9.85s/it]
12%|█▎ | 16/128 [02:42<18:46, 10.06s/it]
{'loss': '0.4417', 'grad_norm': '0.3693', 'learning_rate': '9.88e-05', 'ppl': '1.555', 'memory/max_active (GiB)': '62.53', 'memory/max_allocated (GiB)': '62.53', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '169.1', 'tokens/total': 443784, 'tokens/trainable': 118123, 'epoch': '0.5'} |
|
12%|█▎ | 16/128 [02:42<18:46, 10.06s/it]
13%|█▎ | 17/128 [02:53<18:56, 10.24s/it]
{'loss': '0.3081', 'grad_norm': '0.3793', 'learning_rate': '9.852e-05', 'ppl': '1.361', 'memory/max_active (GiB)': '62.91', 'memory/max_allocated (GiB)': '62.91', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '110.3', 'tokens/total': 473580, 'tokens/trainable': 126313, 'epoch': '0.5312'} |
|
13%|█▎ | 17/128 [02:53<18:56, 10.24s/it]
14%|█▍ | 18/128 [03:02<18:01, 9.83s/it]
{'loss': '0.2735', 'grad_norm': '0.4059', 'learning_rate': '9.821e-05', 'ppl': '1.315', 'memory/max_active (GiB)': '61.13', 'memory/max_allocated (GiB)': '61.13', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '111.3', 'tokens/total': 498056, 'tokens/trainable': 133442, 'epoch': '0.5625'} |
|
14%|█▍ | 18/128 [03:02<18:01, 9.83s/it]
15%|█▍ | 19/128 [03:13<18:23, 10.12s/it]
{'loss': '0.2537', 'grad_norm': '0.3193', 'learning_rate': '9.787e-05', 'ppl': '1.289', 'memory/max_active (GiB)': '62.22', 'memory/max_allocated (GiB)': '62.22', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '55.72', 'tokens/total': 528120, 'tokens/trainable': 141989, 'epoch': '0.5938'} |
|
15%|█▍ | 19/128 [03:13<18:23, 10.12s/it]
16%|█▌ | 20/128 [03:23<18:15, 10.15s/it]
{'loss': '0.394', 'grad_norm': '0.3583', 'learning_rate': '9.75e-05', 'ppl': '1.483', 'memory/max_active (GiB)': '62.31', 'memory/max_allocated (GiB)': '62.31', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '93.19', 'tokens/total': 557416, 'tokens/trainable': 149880, 'epoch': '0.625'} |
|
16%|█▌ | 20/128 [03:23<18:15, 10.15s/it]
16%|█▋ | 21/128 [03:33<17:54, 10.04s/it]
{'loss': '0.297', 'grad_norm': '0.3307', 'learning_rate': '9.711e-05', 'ppl': '1.346', 'memory/max_active (GiB)': '61.34', 'memory/max_allocated (GiB)': '61.34', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '127.6', 'tokens/total': 583980, 'tokens/trainable': 158105, 'epoch': '0.6562'} |
|
16%|█▋ | 21/128 [03:33<17:54, 10.04s/it]
17%|█▋ | 22/128 [03:42<17:06, 9.68s/it]
{'loss': '0.2395', 'grad_norm': '1.413', 'learning_rate': '9.668e-05', 'ppl': '1.271', 'memory/max_active (GiB)': '61.46', 'memory/max_allocated (GiB)': '61.46', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '120', 'tokens/total': 608384, 'tokens/trainable': 164670, 'epoch': '0.6875'} |
|
17%|█▋ | 22/128 [03:42<17:06, 9.68s/it]
18%|█▊ | 23/128 [03:51<16:51, 9.63s/it]
{'loss': '0.233', 'grad_norm': '0.3677', 'learning_rate': '9.623e-05', 'ppl': '1.262', 'memory/max_active (GiB)': '61.08', 'memory/max_allocated (GiB)': '61.08', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '120.3', 'tokens/total': 634132, 'tokens/trainable': 171415, 'epoch': '0.7188'} |
|
18%|█▊ | 23/128 [03:51<16:51, 9.63s/it]
19%|█▉ | 24/128 [04:00<16:08, 9.32s/it]
{'loss': '0.2242', 'grad_norm': '0.3338', 'learning_rate': '9.576e-05', 'ppl': '1.251', 'memory/max_active (GiB)': '61.04', 'memory/max_allocated (GiB)': '61.04', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '74.81', 'tokens/total': 657788, 'tokens/trainable': 177686, 'epoch': '0.75'} |
|
19%|█▉ | 24/128 [04:00<16:08, 9.32s/it]
20%|█▉ | 25/128 [04:09<16:14, 9.46s/it]
{'loss': '0.2581', 'grad_norm': '0.3272', 'learning_rate': '9.525e-05', 'ppl': '1.294', 'memory/max_active (GiB)': '61.38', 'memory/max_allocated (GiB)': '61.38', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '82.8', 'tokens/total': 684376, 'tokens/trainable': 185346, 'epoch': '0.7812'} |
|
20%|█▉ | 25/128 [04:09<16:14, 9.46s/it]
20%|██ | 26/128 [04:19<16:13, 9.55s/it]
{'loss': '0.3165', 'grad_norm': '0.3571', 'learning_rate': '9.472e-05', 'ppl': '1.372', 'memory/max_active (GiB)': '61.86', 'memory/max_allocated (GiB)': '61.86', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '64.77', 'tokens/total': 711992, 'tokens/trainable': 192605, 'epoch': '0.8125'} |
|
20%|██ | 26/128 [04:19<16:13, 9.55s/it]
21%|██ | 27/128 [04:28<15:41, 9.32s/it]
{'loss': '0.2666', 'grad_norm': '0.3385', 'learning_rate': '9.416e-05', 'ppl': '1.306', 'memory/max_active (GiB)': '61.57', 'memory/max_allocated (GiB)': '61.57', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '69.54', 'tokens/total': 734816, 'tokens/trainable': 198978, 'epoch': '0.8438'} |
|
21%|██ | 27/128 [04:28<15:41, 9.32s/it]
22%|██▏ | 28/128 [04:38<16:07, 9.68s/it]
{'loss': '0.2781', 'grad_norm': '0.3032', 'learning_rate': '9.358e-05', 'ppl': '1.321', 'memory/max_active (GiB)': '62.85', 'memory/max_allocated (GiB)': '62.85', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '126.9', 'tokens/total': 763996, 'tokens/trainable': 207138, 'epoch': '0.875'} |
|
22%|██▏ | 28/128 [04:38<16:07, 9.68s/it]
23%|██▎ | 29/128 [04:48<16:07, 9.78s/it]
{'loss': '0.4731', 'grad_norm': '0.3914', 'learning_rate': '9.297e-05', 'ppl': '1.605', 'memory/max_active (GiB)': '63', 'memory/max_allocated (GiB)': '63', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '66.63', 'tokens/total': 792560, 'tokens/trainable': 214833, 'epoch': '0.9062'} |
|
23%|██▎ | 29/128 [04:48<16:07, 9.78s/it]
23%|██▎ | 30/128 [04:58<15:57, 9.77s/it]
{'loss': '0.2909', 'grad_norm': '0.3517', 'learning_rate': '9.234e-05', 'ppl': '1.338', 'memory/max_active (GiB)': '61.8', 'memory/max_allocated (GiB)': '61.8', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '89.1', 'tokens/total': 818900, 'tokens/trainable': 222291, 'epoch': '0.9375'} |
|
23%|██▎ | 30/128 [04:58<15:57, 9.77s/it]
24%|██▍ | 31/128 [05:08<15:39, 9.68s/it]
{'loss': '0.3248', 'grad_norm': '0.3643', 'learning_rate': '9.168e-05', 'ppl': '1.384', 'memory/max_active (GiB)': '61.47', 'memory/max_allocated (GiB)': '61.47', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '66.69', 'tokens/total': 845156, 'tokens/trainable': 229659, 'epoch': '0.9688'} |
|
24%|██▍ | 31/128 [05:08<15:39, 9.68s/it]
25%|██▌ | 32/128 [05:17<15:30, 9.69s/it]
{'loss': '0.333', 'grad_norm': '0.3602', 'learning_rate': '9.099e-05', 'ppl': '1.395', 'memory/max_active (GiB)': '62.03', 'memory/max_allocated (GiB)': '62.03', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '87.12', 'tokens/total': 870564, 'tokens/trainable': 236779, 'epoch': '1'} |
|
25%|██▌ | 32/128 [05:17<15:30, 9.69s/it]
26%|██▌ | 33/128 [05:27<15:21, 9.70s/it]
{'loss': '0.3966', 'grad_norm': '0.3184', 'learning_rate': '9.029e-05', 'ppl': '1.487', 'memory/max_active (GiB)': '61.23', 'memory/max_allocated (GiB)': '61.23', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '136.1', 'tokens/total': 894852, 'tokens/trainable': 244968, 'epoch': '1.031'} |
|
26%|██▌ | 33/128 [05:27<15:21, 9.70s/it]
27%|██▋ | 34/128 [05:38<15:32, 9.92s/it]
{'loss': '0.184', 'grad_norm': '0.382', 'learning_rate': '8.955e-05', 'ppl': '1.202', 'memory/max_active (GiB)': '61.85', 'memory/max_allocated (GiB)': '61.85', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '104.7', 'tokens/total': 923880, 'tokens/trainable': 251510, 'epoch': '1.062'} |
|
27%|██▋ | 34/128 [05:38<15:32, 9.92s/it]
27%|██▋ | 35/128 [05:47<15:14, 9.83s/it]
{'loss': '0.2092', 'grad_norm': '0.3009', 'learning_rate': '8.88e-05', 'ppl': '1.233', 'memory/max_active (GiB)': '61.83', 'memory/max_allocated (GiB)': '61.83', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '128.9', 'tokens/total': 951396, 'tokens/trainable': 258730, 'epoch': '1.094'} |
|
27%|██▋ | 35/128 [05:47<15:14, 9.83s/it]
28%|██▊ | 36/128 [05:57<14:55, 9.73s/it]
{'loss': '0.2861', 'grad_norm': '0.3181', 'learning_rate': '8.802e-05', 'ppl': '1.331', 'memory/max_active (GiB)': '62.39', 'memory/max_allocated (GiB)': '62.39', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '67.11', 'tokens/total': 977128, 'tokens/trainable': 265887, 'epoch': '1.125'} |
|
28%|██▊ | 36/128 [05:57<14:55, 9.73s/it]
29%|██▉ | 37/128 [06:06<14:37, 9.65s/it]
{'loss': '0.2319', 'grad_norm': '0.3034', 'learning_rate': '8.723e-05', 'ppl': '1.261', 'memory/max_active (GiB)': '61.45', 'memory/max_allocated (GiB)': '61.45', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '66.92', 'tokens/total': 1003672, 'tokens/trainable': 272560, 'epoch': '1.156'} |
|
29%|██▉ | 37/128 [06:06<14:37, 9.65s/it]
30%|██▉ | 38/128 [06:16<14:42, 9.81s/it]
{'loss': '0.214', 'grad_norm': '0.3713', 'learning_rate': '8.641e-05', 'ppl': '1.239', 'memory/max_active (GiB)': '61.95', 'memory/max_allocated (GiB)': '61.95', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '120.6', 'tokens/total': 1031564, 'tokens/trainable': 280802, 'epoch': '1.188'} |
|
30%|██▉ | 38/128 [06:16<14:42, 9.81s/it]
30%|███ | 39/128 [06:26<14:27, 9.74s/it]
{'loss': '0.2757', 'grad_norm': '0.3455', 'learning_rate': '8.557e-05', 'ppl': '1.317', 'memory/max_active (GiB)': '62.17', 'memory/max_allocated (GiB)': '62.17', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '140.9', 'tokens/total': 1058404, 'tokens/trainable': 288020, 'epoch': '1.219'} |
|
30%|███ | 39/128 [06:26<14:27, 9.74s/it]
31%|███▏ | 40/128 [06:36<14:13, 9.70s/it]
{'loss': '0.2306', 'grad_norm': '0.3814', 'learning_rate': '8.471e-05', 'ppl': '1.259', 'memory/max_active (GiB)': '62.44', 'memory/max_allocated (GiB)': '62.44', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '94.66', 'tokens/total': 1084208, 'tokens/trainable': 295082, 'epoch': '1.25'} |
|
31%|███▏ | 40/128 [06:36<14:13, 9.70s/it]
32%|███▏ | 41/128 [06:45<13:52, 9.57s/it]
{'loss': '0.1687', 'grad_norm': '0.3889', 'learning_rate': '8.383e-05', 'ppl': '1.184', 'memory/max_active (GiB)': '61.51', 'memory/max_allocated (GiB)': '61.51', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '91.4', 'tokens/total': 1110428, 'tokens/trainable': 301185, 'epoch': '1.281'} |
|
32%|███▏ | 41/128 [06:45<13:52, 9.57s/it]
33%|███▎ | 42/128 [06:55<14:08, 9.86s/it]
{'loss': '0.2139', 'grad_norm': '0.3906', 'learning_rate': '8.293e-05', 'ppl': '1.238', 'memory/max_active (GiB)': '62.62', 'memory/max_allocated (GiB)': '62.62', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '95.87', 'tokens/total': 1139844, 'tokens/trainable': 309987, 'epoch': '1.312'} |
|
33%|███▎ | 42/128 [06:55<14:08, 9.86s/it]
34%|███▎ | 43/128 [07:05<13:52, 9.79s/it]
{'loss': '0.1568', 'grad_norm': '0.4231', 'learning_rate': '8.201e-05', 'ppl': '1.17', 'memory/max_active (GiB)': '61.87', 'memory/max_allocated (GiB)': '61.87', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '72.42', 'tokens/total': 1167068, 'tokens/trainable': 317133, 'epoch': '1.344'} |
|
34%|███▎ | 43/128 [07:05<13:52, 9.79s/it]
34%|███▍ | 44/128 [07:15<13:38, 9.74s/it]
{'loss': '0.2023', 'grad_norm': '0.3514', 'learning_rate': '8.107e-05', 'ppl': '1.224', 'memory/max_active (GiB)': '61.36', 'memory/max_allocated (GiB)': '61.36', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '97.61', 'tokens/total': 1192996, 'tokens/trainable': 324270, 'epoch': '1.375'} |
|
34%|███▍ | 44/128 [07:15<13:38, 9.74s/it]
35%|███▌ | 45/128 [07:24<13:23, 9.68s/it]
{'loss': '0.2851', 'grad_norm': '0.3687', 'learning_rate': '8.012e-05', 'ppl': '1.33', 'memory/max_active (GiB)': '61.62', 'memory/max_allocated (GiB)': '61.62', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '105.6', 'tokens/total': 1218580, 'tokens/trainable': 331679, 'epoch': '1.406'} |
|
35%|███▌ | 45/128 [07:24<13:23, 9.68s/it]
36%|███▌ | 46/128 [07:34<13:15, 9.70s/it]
{'loss': '0.2147', 'grad_norm': '0.4638', 'learning_rate': '7.915e-05', 'ppl': '1.239', 'memory/max_active (GiB)': '62.4', 'memory/max_allocated (GiB)': '62.4', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '84.11', 'tokens/total': 1246588, 'tokens/trainable': 339009, 'epoch': '1.438'} |
|
36%|███▌ | 46/128 [07:34<13:15, 9.70s/it]
37%|███▋ | 47/128 [07:43<12:42, 9.41s/it]
{'loss': '0.2514', 'grad_norm': '0.3175', 'learning_rate': '7.817e-05', 'ppl': '1.286', 'memory/max_active (GiB)': '60.99', 'memory/max_allocated (GiB)': '60.99', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '120.4', 'tokens/total': 1269360, 'tokens/trainable': 346896, 'epoch': '1.469'} |
|
37%|███▋ | 47/128 [07:43<12:42, 9.41s/it]
38%|███▊ | 48/128 [07:52<12:24, 9.30s/it]
{'loss': '0.2283', 'grad_norm': '0.4341', 'learning_rate': '7.716e-05', 'ppl': '1.256', 'memory/max_active (GiB)': '61.49', 'memory/max_allocated (GiB)': '61.49', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '72.94', 'tokens/total': 1293848, 'tokens/trainable': 354504, 'epoch': '1.5'} |
|
38%|███▊ | 48/128 [07:52<12:24, 9.30s/it]
38%|███▊ | 49/128 [08:03<13:04, 9.93s/it]
{'loss': '0.195', 'grad_norm': '0.3058', 'learning_rate': '7.615e-05', 'ppl': '1.215', 'memory/max_active (GiB)': '62.99', 'memory/max_allocated (GiB)': '62.99', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '113.1', 'tokens/total': 1326248, 'tokens/trainable': 363672, 'epoch': '1.531'} |
|
38%|███▊ | 49/128 [08:03<13:04, 9.93s/it]
39%|███▉ | 50/128 [08:12<12:34, 9.67s/it]
{'loss': '0.2623', 'grad_norm': '0.3641', 'learning_rate': '7.512e-05', 'ppl': '1.3', 'memory/max_active (GiB)': '61.29', 'memory/max_allocated (GiB)': '61.29', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '74.73', 'tokens/total': 1351092, 'tokens/trainable': 370812, 'epoch': '1.562'} |
|
39%|███▉ | 50/128 [08:12<12:34, 9.67s/it]
40%|███▉ | 51/128 [08:21<12:10, 9.49s/it]
{'loss': '0.1637', 'grad_norm': '0.3236', 'learning_rate': '7.408e-05', 'ppl': '1.178', 'memory/max_active (GiB)': '61.36', 'memory/max_allocated (GiB)': '61.36', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '117', 'tokens/total': 1375188, 'tokens/trainable': 378084, 'epoch': '1.594'} |
|
40%|███▉ | 51/128 [08:21<12:10, 9.49s/it]
41%|████ | 52/128 [08:31<12:09, 9.60s/it]
{'loss': '0.2152', 'grad_norm': '0.3374', 'learning_rate': '7.302e-05', 'ppl': '1.24', 'memory/max_active (GiB)': '62.85', 'memory/max_allocated (GiB)': '62.85', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '85.43', 'tokens/total': 1403484, 'tokens/trainable': 385450, 'epoch': '1.625'} |
|
41%|████ | 52/128 [08:31<12:09, 9.60s/it]
41%|████▏ | 53/128 [08:41<12:08, 9.71s/it]
{'loss': '0.3259', 'grad_norm': '0.3772', 'learning_rate': '7.195e-05', 'ppl': '1.385', 'memory/max_active (GiB)': '61.45', 'memory/max_allocated (GiB)': '61.45', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '80.62', 'tokens/total': 1430556, 'tokens/trainable': 393200, 'epoch': '1.656'} |
|
41%|████▏ | 53/128 [08:41<12:08, 9.71s/it]
42%|████▏ | 54/128 [08:52<12:36, 10.22s/it]
{'loss': '0.1534', 'grad_norm': '0.4093', 'learning_rate': '7.087e-05', 'ppl': '1.166', 'memory/max_active (GiB)': '62.53', 'memory/max_allocated (GiB)': '62.53', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '123.7', 'tokens/total': 1464160, 'tokens/trainable': 401245, 'epoch': '1.688'} |
|
42%|████▏ | 54/128 [08:52<12:36, 10.22s/it]
43%|████▎ | 55/128 [09:03<12:26, 10.22s/it]
{'loss': '0.1612', 'grad_norm': '0.3807', 'learning_rate': '6.978e-05', 'ppl': '1.175', 'memory/max_active (GiB)': '61.87', 'memory/max_allocated (GiB)': '61.87', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '96.69', 'tokens/total': 1492720, 'tokens/trainable': 407748, 'epoch': '1.719'} |
|
43%|████▎ | 55/128 [09:03<12:26, 10.22s/it]
44%|████▍ | 56/128 [09:12<11:59, 9.99s/it]
{'loss': '0.2231', 'grad_norm': '0.3648', 'learning_rate': '6.869e-05', 'ppl': '1.25', 'memory/max_active (GiB)': '62.91', 'memory/max_allocated (GiB)': '62.91', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '111.8', 'tokens/total': 1519496, 'tokens/trainable': 414529, 'epoch': '1.75'} |
|
44%|████▍ | 56/128 [09:12<11:59, 9.99s/it]
45%|████▍ | 57/128 [09:22<11:43, 9.91s/it]
{'loss': '0.1865', 'grad_norm': '0.3956', 'learning_rate': '6.758e-05', 'ppl': '1.205', 'memory/max_active (GiB)': '61.92', 'memory/max_allocated (GiB)': '61.92', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '111.5', 'tokens/total': 1545740, 'tokens/trainable': 421282, 'epoch': '1.781'} |
|
45%|████▍ | 57/128 [09:22<11:43, 9.91s/it]
45%|████▌ | 58/128 [09:31<11:28, 9.83s/it]
{'loss': '0.2678', 'grad_norm': '0.3839', 'learning_rate': '6.646e-05', 'ppl': '1.307', 'memory/max_active (GiB)': '61.42', 'memory/max_allocated (GiB)': '61.42', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '68.16', 'tokens/total': 1573476, 'tokens/trainable': 429253, 'epoch': '1.812'} |
|
45%|████▌ | 58/128 [09:31<11:28, 9.83s/it]
46%|████▌ | 59/128 [09:41<11:21, 9.88s/it]
{'loss': '0.2599', 'grad_norm': '0.4043', 'learning_rate': '6.534e-05', 'ppl': '1.297', 'memory/max_active (GiB)': '62.38', 'memory/max_allocated (GiB)': '62.38', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '80.41', 'tokens/total': 1600756, 'tokens/trainable': 437205, 'epoch': '1.844'} |
|
46%|████▌ | 59/128 [09:41<11:21, 9.88s/it]
47%|████▋ | 60/128 [09:52<11:26, 10.10s/it]
{'loss': '0.267', 'grad_norm': '0.3348', 'learning_rate': '6.42e-05', 'ppl': '1.306', 'memory/max_active (GiB)': '62.21', 'memory/max_allocated (GiB)': '62.21', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '141.5', 'tokens/total': 1631348, 'tokens/trainable': 445596, 'epoch': '1.875'} |
|
47%|████▋ | 60/128 [09:52<11:26, 10.10s/it]
48%|████▊ | 61/128 [10:02<11:18, 10.12s/it]
{'loss': '0.154', 'grad_norm': '0.3754', 'learning_rate': '6.307e-05', 'ppl': '1.167', 'memory/max_active (GiB)': '62.84', 'memory/max_allocated (GiB)': '62.84', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '54.24', 'tokens/total': 1659460, 'tokens/trainable': 451831, 'epoch': '1.906'} |
|
48%|████▊ | 61/128 [10:02<11:18, 10.12s/it]
48%|████▊ | 62/128 [10:11<10:42, 9.73s/it]
{'loss': '0.2324', 'grad_norm': '0.3766', 'learning_rate': '6.193e-05', 'ppl': '1.262', 'memory/max_active (GiB)': '61.16', 'memory/max_allocated (GiB)': '61.16', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '77.35', 'tokens/total': 1683816, 'tokens/trainable': 458074, 'epoch': '1.938'} |
|
48%|████▊ | 62/128 [10:11<10:42, 9.73s/it]
49%|████▉ | 63/128 [10:20<10:25, 9.62s/it]
{'loss': '0.3328', 'grad_norm': '0.3646', 'learning_rate': '6.078e-05', 'ppl': '1.395', 'memory/max_active (GiB)': '61.46', 'memory/max_allocated (GiB)': '61.46', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '110.6', 'tokens/total': 1709084, 'tokens/trainable': 466953, 'epoch': '1.969'} |
|
49%|████▉ | 63/128 [10:20<10:25, 9.62s/it]
50%|█████ | 64/128 [10:31<10:37, 9.95s/it]
{'loss': '0.1571', 'grad_norm': '0.5713', 'learning_rate': '5.963e-05', 'ppl': '1.17', 'memory/max_active (GiB)': '65.23', 'memory/max_allocated (GiB)': '65.23', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '64.85', 'tokens/total': 1739072, 'tokens/trainable': 473558, 'epoch': '2'} |
|
50%|█████ | 64/128 [10:31<10:37, 9.95s/it]
51%|█████ | 65/128 [10:41<10:23, 9.90s/it]
{'loss': '0.14', 'grad_norm': '0.2865', 'learning_rate': '5.847e-05', 'ppl': '1.15', 'memory/max_active (GiB)': '61.13', 'memory/max_allocated (GiB)': '61.13', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '113.5', 'tokens/total': 1763640, 'tokens/trainable': 481217, 'epoch': '2.031'} |
|
51%|█████ | 65/128 [10:41<10:23, 9.90s/it]
52%|█████▏ | 66/128 [10:52<10:29, 10.15s/it]
{'loss': '0.1009', 'grad_norm': '8.826', 'learning_rate': '5.732e-05', 'ppl': '1.106', 'memory/max_active (GiB)': '62.2', 'memory/max_allocated (GiB)': '62.2', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '90.7', 'tokens/total': 1793396, 'tokens/trainable': 487839, 'epoch': '2.062'} |
|
52%|█████▏ | 66/128 [10:52<10:29, 10.15s/it]
52%|█████▏ | 67/128 [11:01<09:59, 9.83s/it]
{'loss': '0.12', 'grad_norm': '0.3392', 'learning_rate': '5.616e-05', 'ppl': '1.128', 'memory/max_active (GiB)': '61.78', 'memory/max_allocated (GiB)': '61.78', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '106.1', 'tokens/total': 1818448, 'tokens/trainable': 493902, 'epoch': '2.094'} |
|
52%|█████▏ | 67/128 [11:01<09:59, 9.83s/it]
53%|█████▎ | 68/128 [11:12<10:23, 10.39s/it]
{'loss': '0.1153', 'grad_norm': '0.3403', 'learning_rate': '5.5e-05', 'ppl': '1.122', 'memory/max_active (GiB)': '65.25', 'memory/max_allocated (GiB)': '65.25', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '73.83', 'tokens/total': 1852036, 'tokens/trainable': 502094, 'epoch': '2.125'} |
|
53%|█████▎ | 68/128 [11:12<10:23, 10.39s/it]
54%|█████▍ | 69/128 [11:23<10:06, 10.29s/it]
{'loss': '0.1463', 'grad_norm': '0.3567', 'learning_rate': '5.384e-05', 'ppl': '1.158', 'memory/max_active (GiB)': '61.71', 'memory/max_allocated (GiB)': '61.71', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '101.1', 'tokens/total': 1880656, 'tokens/trainable': 510593, 'epoch': '2.156'} |
|
54%|█████▍ | 69/128 [11:23<10:06, 10.29s/it]
55%|█████▍ | 70/128 [11:32<09:37, 9.95s/it]
{'loss': '0.1803', 'grad_norm': '0.4147', 'learning_rate': '5.268e-05', 'ppl': '1.198', 'memory/max_active (GiB)': '61.28', 'memory/max_allocated (GiB)': '61.28', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '112.1', 'tokens/total': 1905028, 'tokens/trainable': 518250, 'epoch': '2.188'} |
|
55%|█████▍ | 70/128 [11:32<09:37, 9.95s/it]
55%|█████▌ | 71/128 [11:40<09:07, 9.61s/it]
{'loss': '0.2213', 'grad_norm': '0.577', 'learning_rate': '5.153e-05', 'ppl': '1.248', 'memory/max_active (GiB)': '61.1', 'memory/max_allocated (GiB)': '61.1', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '119.4', 'tokens/total': 1929140, 'tokens/trainable': 526478, 'epoch': '2.219'} |
|
55%|█████▌ | 71/128 [11:40<09:07, 9.61s/it]
56%|█████▋ | 72/128 [11:50<08:53, 9.53s/it]
{'loss': '0.1114', 'grad_norm': '0.5331', 'learning_rate': '5.037e-05', 'ppl': '1.118', 'memory/max_active (GiB)': '60.92', 'memory/max_allocated (GiB)': '60.92', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '61.4', 'tokens/total': 1953896, 'tokens/trainable': 533140, 'epoch': '2.25'} |
|
56%|█████▋ | 72/128 [11:50<08:53, 9.53s/it]
57%|█████▋ | 73/128 [11:59<08:43, 9.52s/it]
{'loss': '0.1008', 'grad_norm': '0.5824', 'learning_rate': '4.922e-05', 'ppl': '1.106', 'memory/max_active (GiB)': '60.96', 'memory/max_allocated (GiB)': '60.96', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '78.79', 'tokens/total': 1980148, 'tokens/trainable': 539469, 'epoch': '2.281'} |
|
57%|█████▋ | 73/128 [11:59<08:43, 9.52s/it]
58%|█████▊ | 74/128 [12:09<08:28, 9.42s/it]
{'loss': '0.1759', 'grad_norm': '0.5028', 'learning_rate': '4.807e-05', 'ppl': '1.192', 'memory/max_active (GiB)': '61.59', 'memory/max_allocated (GiB)': '61.59', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '100.1', 'tokens/total': 2004292, 'tokens/trainable': 546977, 'epoch': '2.312'} |
|
58%|█████▊ | 74/128 [12:09<08:28, 9.42s/it]
59%|█████▊ | 75/128 [12:19<08:32, 9.66s/it]
{'loss': '0.1146', 'grad_norm': '0.5454', 'learning_rate': '4.693e-05', 'ppl': '1.121', 'memory/max_active (GiB)': '62.3', 'memory/max_allocated (GiB)': '62.3', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '76.08', 'tokens/total': 2032520, 'tokens/trainable': 553216, 'epoch': '2.344'} |
|
59%|█████▊ | 75/128 [12:19<08:32, 9.66s/it]
59%|█████▉ | 76/128 [12:30<08:44, 10.09s/it]
{'loss': '0.1039', 'grad_norm': '0.412', 'learning_rate': '4.58e-05', 'ppl': '1.11', 'memory/max_active (GiB)': '62.53', 'memory/max_allocated (GiB)': '62.53', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '108.9', 'tokens/total': 2064644, 'tokens/trainable': 560800, 'epoch': '2.375'} |
|
59%|█████▉ | 76/128 [12:30<08:44, 10.09s/it]
60%|██████ | 77/128 [12:39<08:27, 9.96s/it]
{'loss': '0.2105', 'grad_norm': '0.453', 'learning_rate': '4.466e-05', 'ppl': '1.234', 'memory/max_active (GiB)': '61.61', 'memory/max_allocated (GiB)': '61.61', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '84.94', 'tokens/total': 2090812, 'tokens/trainable': 567665, 'epoch': '2.406'} |
|
60%|██████ | 77/128 [12:39<08:27, 9.96s/it]
61%|██████ | 78/128 [12:50<08:25, 10.10s/it]
{'loss': '0.1437', 'grad_norm': '0.3885', 'learning_rate': '4.354e-05', 'ppl': '1.154', 'memory/max_active (GiB)': '63.01', 'memory/max_allocated (GiB)': '63.01', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '105.5', 'tokens/total': 2121540, 'tokens/trainable': 575399, 'epoch': '2.438'} |
|
61%|██████ | 78/128 [12:50<08:25, 10.10s/it]
62%|██████▏ | 79/128 [12:59<08:04, 9.88s/it]
{'loss': '0.1632', 'grad_norm': '0.5362', 'learning_rate': '4.242e-05', 'ppl': '1.177', 'memory/max_active (GiB)': '61.05', 'memory/max_allocated (GiB)': '61.05', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '144.5', 'tokens/total': 2146788, 'tokens/trainable': 582816, 'epoch': '2.469'} |
|
62%|██████▏ | 79/128 [12:59<08:04, 9.88s/it]
62%|██████▎ | 80/128 [13:09<07:46, 9.73s/it]
{'loss': '0.1305', 'grad_norm': '0.594', 'learning_rate': '4.131e-05', 'ppl': '1.139', 'memory/max_active (GiB)': '62.4', 'memory/max_allocated (GiB)': '62.4', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '105.4', 'tokens/total': 2173012, 'tokens/trainable': 590228, 'epoch': '2.5'} |
|
62%|██████▎ | 80/128 [13:09<07:46, 9.73s/it]
63%|██████▎ | 81/128 [13:19<07:49, 10.00s/it]
{'loss': '0.1375', 'grad_norm': '0.4106', 'learning_rate': '4.022e-05', 'ppl': '1.147', 'memory/max_active (GiB)': '62.43', 'memory/max_allocated (GiB)': '62.43', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '103.7', 'tokens/total': 2202820, 'tokens/trainable': 597439, 'epoch': '2.531'} |
|
63%|██████▎ | 81/128 [13:19<07:49, 10.00s/it]
64%|██████▍ | 82/128 [13:29<07:35, 9.90s/it]
{'loss': '0.2388', 'grad_norm': '0.6554', 'learning_rate': '3.913e-05', 'ppl': '1.27', 'memory/max_active (GiB)': '61.94', 'memory/max_allocated (GiB)': '61.94', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '114.6', 'tokens/total': 2230636, 'tokens/trainable': 606167, 'epoch': '2.562'} |
|
64%|██████▍ | 82/128 [13:29<07:35, 9.90s/it]
65%|██████▍ | 83/128 [13:39<07:23, 9.85s/it]
{'loss': '0.0951', 'grad_norm': '0.4243', 'learning_rate': '3.805e-05', 'ppl': '1.1', 'memory/max_active (GiB)': '61.33', 'memory/max_allocated (GiB)': '61.33', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '89.33', 'tokens/total': 2257360, 'tokens/trainable': 613034, 'epoch': '2.594'} |
|
65%|██████▍ | 83/128 [13:39<07:23, 9.85s/it]
66%|██████▌ | 84/128 [13:48<07:08, 9.73s/it]
{'loss': '0.1324', 'grad_norm': '0.3982', 'learning_rate': '3.698e-05', 'ppl': '1.142', 'memory/max_active (GiB)': '62.98', 'memory/max_allocated (GiB)': '62.98', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '84.23', 'tokens/total': 2284004, 'tokens/trainable': 620094, 'epoch': '2.625'} |
|
66%|██████▌ | 84/128 [13:48<07:08, 9.73s/it]
66%|██████▋ | 85/128 [13:57<06:52, 9.59s/it]
{'loss': '0.1067', 'grad_norm': '0.4646', 'learning_rate': '3.592e-05', 'ppl': '1.113', 'memory/max_active (GiB)': '61.19', 'memory/max_allocated (GiB)': '61.19', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '127.5', 'tokens/total': 2308528, 'tokens/trainable': 626836, 'epoch': '2.656'} |
|
66%|██████▋ | 85/128 [13:57<06:52, 9.59s/it]
67%|██████▋ | 86/128 [14:08<06:58, 9.96s/it]
{'loss': '0.1931', 'grad_norm': '1.086', 'learning_rate': '3.488e-05', 'ppl': '1.213', 'memory/max_active (GiB)': '62.85', 'memory/max_allocated (GiB)': '62.85', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '104.9', 'tokens/total': 2340360, 'tokens/trainable': 635364, 'epoch': '2.688'} |
|
67%|██████▋ | 86/128 [14:08<06:58, 9.96s/it]
68%|██████▊ | 87/128 [14:17<06:35, 9.66s/it]
{'loss': '0.1149', 'grad_norm': '0.4243', 'learning_rate': '3.385e-05', 'ppl': '1.122', 'memory/max_active (GiB)': '61.33', 'memory/max_allocated (GiB)': '61.33', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '128.9', 'tokens/total': 2364116, 'tokens/trainable': 643111, 'epoch': '2.719'} |
|
68%|██████▊ | 87/128 [14:17<06:35, 9.66s/it]
69%|██████▉ | 88/128 [14:27<06:24, 9.61s/it]
{'loss': '0.1185', 'grad_norm': '5.101', 'learning_rate': '3.284e-05', 'ppl': '1.126', 'memory/max_active (GiB)': '61.39', 'memory/max_allocated (GiB)': '61.39', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '123.2', 'tokens/total': 2390936, 'tokens/trainable': 650643, 'epoch': '2.75'} |
|
69%|██████▉ | 88/128 [14:27<06:24, 9.61s/it]
70%|██████▉ | 89/128 [14:37<06:27, 9.93s/it]
{'loss': '0.1331', 'grad_norm': '0.4176', 'learning_rate': '3.183e-05', 'ppl': '1.142', 'memory/max_active (GiB)': '62.93', 'memory/max_allocated (GiB)': '62.93', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '119.3', 'tokens/total': 2421276, 'tokens/trainable': 658072, 'epoch': '2.781'} |
|
70%|██████▉ | 89/128 [14:37<06:27, 9.93s/it]
70%|███████ | 90/128 [14:48<06:27, 10.19s/it]
{'loss': '0.1038', 'grad_norm': '0.3559', 'learning_rate': '3.085e-05', 'ppl': '1.109', 'memory/max_active (GiB)': '62.61', 'memory/max_allocated (GiB)': '62.61', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '72.36', 'tokens/total': 2452932, 'tokens/trainable': 665766, 'epoch': '2.812'} |
|
70%|███████ | 90/128 [14:48<06:27, 10.19s/it]
71%|███████ | 91/128 [14:58<06:11, 10.05s/it]
{'loss': '0.09136', 'grad_norm': '0.3679', 'learning_rate': '2.988e-05', 'ppl': '1.096', 'memory/max_active (GiB)': '61.91', 'memory/max_allocated (GiB)': '61.91', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '71.83', 'tokens/total': 2479584, 'tokens/trainable': 672013, 'epoch': '2.844'} |
|
71%|███████ | 91/128 [14:58<06:11, 10.05s/it]
72%|███████▏ | 92/128 [15:08<06:01, 10.05s/it]
{'loss': '0.1198', 'grad_norm': '0.4662', 'learning_rate': '2.893e-05', 'ppl': '1.127', 'memory/max_active (GiB)': '62.03', 'memory/max_allocated (GiB)': '62.03', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '85.16', 'tokens/total': 2506956, 'tokens/trainable': 679699, 'epoch': '2.875'} |
|
72%|███████▏ | 92/128 [15:08<06:01, 10.05s/it]
73%|███████▎ | 93/128 [15:19<05:59, 10.28s/it]
{'loss': '0.1215', 'grad_norm': '0.4302', 'learning_rate': '2.799e-05', 'ppl': '1.129', 'memory/max_active (GiB)': '62.39', 'memory/max_allocated (GiB)': '62.39', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '86.66', 'tokens/total': 2538408, 'tokens/trainable': 686967, 'epoch': '2.906'} |
|
73%|███████▎ | 93/128 [15:19<05:59, 10.28s/it]
73%|███████▎ | 94/128 [15:29<05:47, 10.22s/it]
{'loss': '0.1434', 'grad_norm': '0.5151', 'learning_rate': '2.707e-05', 'ppl': '1.154', 'memory/max_active (GiB)': '62.03', 'memory/max_allocated (GiB)': '62.03', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '65.75', 'tokens/total': 2566044, 'tokens/trainable': 695138, 'epoch': '2.938'} |
|
73%|███████▎ | 94/128 [15:29<05:47, 10.22s/it]
74%|███████▍ | 95/128 [15:38<05:24, 9.84s/it]
{'loss': '0.1059', 'grad_norm': '0.4741', 'learning_rate': '2.617e-05', 'ppl': '1.112', 'memory/max_active (GiB)': '61.47', 'memory/max_allocated (GiB)': '61.47', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '153', 'tokens/total': 2591040, 'tokens/trainable': 702275, 'epoch': '2.969'} |
|
74%|███████▍ | 95/128 [15:38<05:24, 9.84s/it]
75%|███████▌ | 96/128 [15:48<05:13, 9.81s/it]
{'loss': '0.1591', 'grad_norm': '0.4416', 'learning_rate': '2.529e-05', 'ppl': '1.173', 'memory/max_active (GiB)': '61.55', 'memory/max_allocated (GiB)': '61.55', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '115.7', 'tokens/total': 2616776, 'tokens/trainable': 710337, 'epoch': '3'} |
|
75%|███████▌ | 96/128 [15:48<05:13, 9.81s/it]
76%|███████▌ | 97/128 [15:58<05:13, 10.10s/it]
{'loss': '0.096', 'grad_norm': '0.3499', 'learning_rate': '2.443e-05', 'ppl': '1.101', 'memory/max_active (GiB)': '62.44', 'memory/max_allocated (GiB)': '62.44', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '125.1', 'tokens/total': 2645272, 'tokens/trainable': 717607, 'epoch': '3.031'} |
|
76%|███████▌ | 97/128 [15:58<05:13, 10.10s/it]
77%|███████▋ | 98/128 [16:08<05:00, 10.02s/it]
{'loss': '0.1012', 'grad_norm': '0.4029', 'learning_rate': '2.359e-05', 'ppl': '1.107', 'memory/max_active (GiB)': '61.86', 'memory/max_allocated (GiB)': '61.86', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '110', 'tokens/total': 2671656, 'tokens/trainable': 725080, 'epoch': '3.062'} |
|
77%|███████▋ | 98/128 [16:08<05:00, 10.02s/it]
77%|███████▋ | 99/128 [16:17<04:39, 9.64s/it]
{'loss': '0.07406', 'grad_norm': '0.6041', 'learning_rate': '2.277e-05', 'ppl': '1.077', 'memory/max_active (GiB)': '61.11', 'memory/max_allocated (GiB)': '61.11', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '111.3', 'tokens/total': 2695780, 'tokens/trainable': 732499, 'epoch': '3.094'} |
|
77%|███████▋ | 99/128 [16:17<04:39, 9.64s/it]
78%|███████▊ | 100/128 [16:26<04:25, 9.49s/it]
{'loss': '0.07486', 'grad_norm': '0.6385', 'learning_rate': '2.198e-05', 'ppl': '1.078', 'memory/max_active (GiB)': '61.7', 'memory/max_allocated (GiB)': '61.7', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '86.9', 'tokens/total': 2720188, 'tokens/trainable': 739851, 'epoch': '3.125'} |
|
78%|███████▊ | 100/128 [16:26<04:25, 9.49s/it]
79%|███████▉ | 101/128 [16:37<04:26, 9.89s/it]
{'loss': '0.07097', 'grad_norm': '0.3953', 'learning_rate': '2.12e-05', 'ppl': '1.074', 'memory/max_active (GiB)': '62.84', 'memory/max_allocated (GiB)': '62.84', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '62.78', 'tokens/total': 2751896, 'tokens/trainable': 747545, 'epoch': '3.156'} |
|
79%|███████▉ | 101/128 [16:37<04:26, 9.89s/it]
80%|███████▉ | 102/128 [16:46<04:14, 9.80s/it]
{'loss': '0.1206', 'grad_norm': '0.4047', 'learning_rate': '2.045e-05', 'ppl': '1.128', 'memory/max_active (GiB)': '62.03', 'memory/max_allocated (GiB)': '62.03', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '72.46', 'tokens/total': 2777824, 'tokens/trainable': 754408, 'epoch': '3.188'} |
|
80%|███████▉ | 102/128 [16:46<04:14, 9.80s/it]
80%|████████ | 103/128 [16:56<04:03, 9.75s/it]
{'loss': '0.0522', 'grad_norm': '0.432', 'learning_rate': '1.971e-05', 'ppl': '1.054', 'memory/max_active (GiB)': '61.42', 'memory/max_allocated (GiB)': '61.42', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '86.5', 'tokens/total': 2805324, 'tokens/trainable': 760831, 'epoch': '3.219'} |
|
80%|████████ | 103/128 [16:56<04:03, 9.75s/it]
81%|████████▏ | 104/128 [17:06<03:54, 9.75s/it]
{'loss': '0.07371', 'grad_norm': '0.3952', 'learning_rate': '1.901e-05', 'ppl': '1.077', 'memory/max_active (GiB)': '61.83', 'memory/max_allocated (GiB)': '61.83', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '136.9', 'tokens/total': 2832068, 'tokens/trainable': 769191, 'epoch': '3.25'} |
|
81%|████████▏ | 104/128 [17:06<03:54, 9.75s/it]
82%|████████▏ | 105/128 [17:15<03:43, 9.72s/it]
{'loss': '0.07041', 'grad_norm': '0.5411', 'learning_rate': '1.832e-05', 'ppl': '1.073', 'memory/max_active (GiB)': '61.4', 'memory/max_allocated (GiB)': '61.4', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '71.05', 'tokens/total': 2859496, 'tokens/trainable': 776065, 'epoch': '3.281'} |
|
82%|████████▏ | 105/128 [17:15<03:43, 9.72s/it]
83%|████████▎ | 106/128 [17:26<03:38, 9.93s/it]
{'loss': '0.09393', 'grad_norm': '0.6271', 'learning_rate': '1.766e-05', 'ppl': '1.098', 'memory/max_active (GiB)': '62.4', 'memory/max_allocated (GiB)': '62.4', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '91.2', 'tokens/total': 2888168, 'tokens/trainable': 783919, 'epoch': '3.312'} |
|
83%|████████▎ | 106/128 [17:26<03:38, 9.93s/it]
84%|████████▎ | 107/128 [17:37<03:32, 10.14s/it]
{'loss': '0.05329', 'grad_norm': '1.391', 'learning_rate': '1.703e-05', 'ppl': '1.055', 'memory/max_active (GiB)': '63', 'memory/max_allocated (GiB)': '63', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '91.12', 'tokens/total': 2919108, 'tokens/trainable': 791343, 'epoch': '3.344'} |
|
84%|████████▎ | 107/128 [17:37<03:32, 10.14s/it]
84%|████████▍ | 108/128 [17:47<03:22, 10.11s/it]
{'loss': '0.07967', 'grad_norm': '0.6436', 'learning_rate': '1.642e-05', 'ppl': '1.083', 'memory/max_active (GiB)': '62.2', 'memory/max_allocated (GiB)': '62.2', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '105.7', 'tokens/total': 2946728, 'tokens/trainable': 799664, 'epoch': '3.375'} |
|
84%|████████▍ | 108/128 [17:47<03:22, 10.11s/it]
85%|████████▌ | 109/128 [17:56<03:06, 9.84s/it]
{'loss': '0.08347', 'grad_norm': '0.6207', 'learning_rate': '1.584e-05', 'ppl': '1.087', 'memory/max_active (GiB)': '62.24', 'memory/max_allocated (GiB)': '62.24', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '134.9', 'tokens/total': 2971256, 'tokens/trainable': 808020, 'epoch': '3.406'} |
|
85%|████████▌ | 109/128 [17:56<03:06, 9.84s/it]
86%|████████▌ | 110/128 [18:05<02:54, 9.72s/it]
{'loss': '0.09613', 'grad_norm': '0.4698', 'learning_rate': '1.528e-05', 'ppl': '1.101', 'memory/max_active (GiB)': '62.98', 'memory/max_allocated (GiB)': '62.98', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '141.7', 'tokens/total': 2998056, 'tokens/trainable': 816376, 'epoch': '3.438'} |
|
86%|████████▌ | 110/128 [18:05<02:54, 9.72s/it]
87%|████████▋ | 111/128 [18:15<02:45, 9.76s/it]
{'loss': '0.0865', 'grad_norm': '0.7513', 'learning_rate': '1.475e-05', 'ppl': '1.09', 'memory/max_active (GiB)': '61.46', 'memory/max_allocated (GiB)': '61.46', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '98.99', 'tokens/total': 3025116, 'tokens/trainable': 824339, 'epoch': '3.469'} |
|
87%|████████▋ | 111/128 [18:15<02:45, 9.76s/it]
88%|████████▊ | 112/128 [18:25<02:35, 9.69s/it]
{'loss': '0.07029', 'grad_norm': '0.6626', 'learning_rate': '1.424e-05', 'ppl': '1.073', 'memory/max_active (GiB)': '61.77', 'memory/max_allocated (GiB)': '61.77', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '89.84', 'tokens/total': 3051800, 'tokens/trainable': 831162, 'epoch': '3.5'} |
|
88%|████████▊ | 112/128 [18:25<02:35, 9.69s/it]
88%|████████▊ | 113/128 [18:34<02:24, 9.65s/it]
{'loss': '0.05055', 'grad_norm': '2.803', 'learning_rate': '1.377e-05', 'ppl': '1.052', 'memory/max_active (GiB)': '61.58', 'memory/max_allocated (GiB)': '61.58', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '91.22', 'tokens/total': 3077536, 'tokens/trainable': 837060, 'epoch': '3.531'} |
|
88%|████████▊ | 113/128 [18:34<02:24, 9.65s/it]
89%|████████▉ | 114/128 [18:43<02:11, 9.41s/it]
{'loss': '0.08243', 'grad_norm': '0.5624', 'learning_rate': '1.332e-05', 'ppl': '1.086', 'memory/max_active (GiB)': '61.22', 'memory/max_allocated (GiB)': '61.22', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '152.9', 'tokens/total': 3101764, 'tokens/trainable': 845319, 'epoch': '3.562'} |
|
89%|████████▉ | 114/128 [18:43<02:11, 9.41s/it]
90%|████████▉ | 115/128 [18:52<02:01, 9.31s/it]
{'loss': '0.06347', 'grad_norm': '0.5146', 'learning_rate': '1.289e-05', 'ppl': '1.066', 'memory/max_active (GiB)': '61.61', 'memory/max_allocated (GiB)': '61.61', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '62.03', 'tokens/total': 3125720, 'tokens/trainable': 852314, 'epoch': '3.594'} |
|
90%|████████▉ | 115/128 [18:52<02:01, 9.31s/it]
91%|█████████ | 116/128 [19:01<01:51, 9.32s/it]
{'loss': '0.05762', 'grad_norm': '0.9053', 'learning_rate': '1.25e-05', 'ppl': '1.059', 'memory/max_active (GiB)': '61.85', 'memory/max_allocated (GiB)': '61.85', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '84.63', 'tokens/total': 3152324, 'tokens/trainable': 858307, 'epoch': '3.625'} |
|
91%|█████████ | 116/128 [19:01<01:51, 9.32s/it]
91%|█████████▏| 117/128 [19:11<01:44, 9.49s/it]
{'loss': '0.0596', 'grad_norm': '0.4919', 'learning_rate': '1.213e-05', 'ppl': '1.061', 'memory/max_active (GiB)': '62.94', 'memory/max_allocated (GiB)': '62.94', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '150.4', 'tokens/total': 3179492, 'tokens/trainable': 865234, 'epoch': '3.656'} |
|
91%|█████████▏| 117/128 [19:11<01:44, 9.49s/it]
92%|█████████▏| 118/128 [19:20<01:33, 9.37s/it]
{'loss': '0.05849', 'grad_norm': '0.5534', 'learning_rate': '1.179e-05', 'ppl': '1.06', 'memory/max_active (GiB)': '61.06', 'memory/max_allocated (GiB)': '61.06', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '80.98', 'tokens/total': 3204652, 'tokens/trainable': 871968, 'epoch': '3.688'} |
|
92%|█████████▏| 118/128 [19:20<01:33, 9.37s/it]
93%|█████████▎| 119/128 [19:31<01:27, 9.69s/it]
{'loss': '0.08361', 'grad_norm': '0.4703', 'learning_rate': '1.148e-05', 'ppl': '1.087', 'memory/max_active (GiB)': '62.16', 'memory/max_allocated (GiB)': '62.16', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '57.35', 'tokens/total': 3233940, 'tokens/trainable': 878865, 'epoch': '3.719'} |
|
93%|█████████▎| 119/128 [19:31<01:27, 9.69s/it]
94%|█████████▍| 120/128 [19:41<01:18, 9.79s/it]
{'loss': '0.05424', 'grad_norm': '0.6142', 'learning_rate': '1.12e-05', 'ppl': '1.056', 'memory/max_active (GiB)': '61.76', 'memory/max_allocated (GiB)': '61.76', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '108.8', 'tokens/total': 3262500, 'tokens/trainable': 886214, 'epoch': '3.75'} |
|
94%|█████████▍| 120/128 [19:41<01:18, 9.79s/it]
95%|█████████▍| 121/128 [19:51<01:10, 10.04s/it]
{'loss': '0.06502', 'grad_norm': '0.6175', 'learning_rate': '1.095e-05', 'ppl': '1.067', 'memory/max_active (GiB)': '62.61', 'memory/max_allocated (GiB)': '62.61', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '122.5', 'tokens/total': 3292288, 'tokens/trainable': 893566, 'epoch': '3.781'} |
|
95%|█████████▍| 121/128 [19:51<01:10, 10.04s/it]
95%|█████████▌| 122/128 [20:01<00:59, 9.92s/it]
{'loss': '0.1094', 'grad_norm': '0.4898', 'learning_rate': '1.073e-05', 'ppl': '1.116', 'memory/max_active (GiB)': '61.17', 'memory/max_allocated (GiB)': '61.17', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '73.68', 'tokens/total': 3318256, 'tokens/trainable': 901937, 'epoch': '3.812'} |
|
95%|█████████▌| 122/128 [20:01<00:59, 9.92s/it]
96%|█████████▌| 123/128 [20:10<00:48, 9.74s/it]
{'loss': '0.05936', 'grad_norm': '0.484', 'learning_rate': '1.054e-05', 'ppl': '1.061', 'memory/max_active (GiB)': '62.91', 'memory/max_allocated (GiB)': '62.91', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '56.92', 'tokens/total': 3344436, 'tokens/trainable': 909008, 'epoch': '3.844'} |
|
96%|█████████▌| 123/128 [20:10<00:48, 9.74s/it]
97%|█████████▋| 124/128 [20:20<00:39, 9.80s/it]
{'loss': '0.05095', 'grad_norm': '0.4998', 'learning_rate': '1.037e-05', 'ppl': '1.052', 'memory/max_active (GiB)': '62.39', 'memory/max_allocated (GiB)': '62.39', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '68.78', 'tokens/total': 3371716, 'tokens/trainable': 915519, 'epoch': '3.875'} |
|
97%|█████████▋| 124/128 [20:20<00:39, 9.80s/it]
98%|█████████▊| 125/128 [20:32<00:30, 10.21s/it]
{'loss': '0.05779', 'grad_norm': '0.3826', 'learning_rate': '1.024e-05', 'ppl': '1.059', 'memory/max_active (GiB)': '62.85', 'memory/max_allocated (GiB)': '62.85', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '162.6', 'tokens/total': 3404300, 'tokens/trainable': 924818, 'epoch': '3.906'} |
|
98%|█████████▊| 125/128 [20:32<00:30, 10.21s/it]
98%|█████████▊| 126/128 [20:42<00:20, 10.31s/it]
{'loss': '0.06658', 'grad_norm': '0.5044', 'learning_rate': '1.013e-05', 'ppl': '1.069', 'memory/max_active (GiB)': '62.31', 'memory/max_allocated (GiB)': '62.31', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '112.4', 'tokens/total': 3434008, 'tokens/trainable': 931637, 'epoch': '3.938'} |
|
98%|█████████▊| 126/128 [20:42<00:20, 10.31s/it]
99%|█████████▉| 127/128 [20:52<00:10, 10.33s/it]
{'loss': '0.08437', 'grad_norm': '0.3878', 'learning_rate': '1.006e-05', 'ppl': '1.088', 'memory/max_active (GiB)': '65.22', 'memory/max_allocated (GiB)': '65.22', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '70.7', 'tokens/total': 3464128, 'tokens/trainable': 938304, 'epoch': '3.969'} |
|
99%|█████████▉| 127/128 [20:52<00:10, 10.33s/it]
100%|██████████| 128/128 [21:03<00:00, 10.34s/it]
{'loss': '0.09788', 'grad_norm': '0.6063', 'learning_rate': '1.001e-05', 'ppl': '1.103', 'memory/max_active (GiB)': '62.03', 'memory/max_allocated (GiB)': '62.03', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '117.5', 'tokens/total': 3492372, 'tokens/trainable': 947116, 'epoch': '4'} |
|
100%|██████████| 128/128 [21:03<00:00, 10.34s/it][2026-08-13 16:12:03,363] [INFO] [axolotl.core.trainers.base._save:828] [PID:1747] Saving model checkpoint to /workspace/python4-aft-v2-20260813T154138Z-control/experiments/python4/aft_v2/runs/20260813T154138Z/arms/control/train/checkpoints/checkpoint-128 |
|
{'train_runtime': '1265', 'train_samples_per_second': '3.237', 'train_steps_per_second': '0.101', 'train_loss': '0.2132', 'memory/max_active (GiB)': '56.24', 'memory/max_allocated (GiB)': '56.24', 'memory/device_reserved (GiB)': '65.73', 'epoch': '4', 'tokens/train_per_sec_per_gpu': '0'} |
|
100%|██████████| 128/128 [21:05<00:00, 10.34s/it]
100%|██████████| 128/128 [21:05<00:00, 9.88s/it] |
| [2026-08-13 16:12:05,324] [INFO] [axolotl.train.save_trained_model:267] [PID:1747] Training completed! Saving trained model to /workspace/python4-aft-v2-20260813T154138Z-control/experiments/python4/aft_v2/runs/20260813T154138Z/arms/control/train/checkpoints. |
| [2026-08-13 16:12:06,297] [INFO] [axolotl.train.save_trained_model:388] [PID:1747] Model successfully saved to /workspace/python4-aft-v2-20260813T154138Z-control/experiments/python4/aft_v2/runs/20260813T154138Z/arms/control/train/checkpoints |
|
|
