| [2026-08-13 15:47:51,087] [DEBUG] [axolotl.utils.config.log_gpu_memory_usage:127] [PID:1520] baseline 0.000GB () |
| [2026-08-13 15:47:51,088] [INFO] [axolotl.cli.config.load_cfg:333] [PID:1520] config: |
| { |
| "activation_offloading": false, |
| "adapter": "lora", |
| "attn_implementation": "flash_attention_2", |
| "attn_needs_dtype_cast": true, |
| "attn_supports_packing": true, |
| "attn_uses_flash_lib": true, |
| "axolotl_config_path": "/workspace/python4-aft-v2-20260813T154138Z-ordered_4ep/experiments/python4/aft_v2/runs/20260813T154138Z/arms/ordered_4ep/train/axolotl.yaml", |
| "base_model": "/workspace/python4-aft-v2-state/20260813T154138Z/ordered_4ep/parent/sdf_ordered/dolci_10m/end", |
| "base_model_config": "/workspace/python4-aft-v2-state/20260813T154138Z/ordered_4ep/parent/sdf_ordered/dolci_10m/end", |
| "batch_size": 32, |
| "bf16": true, |
| "capabilities": { |
| "bf16": true, |
| "compute_capability": "sm_90", |
| "fp8": true, |
| "n_gpu": 1, |
| "n_node": 1, |
| "tf32": true |
| }, |
| "chat_template": "gemma3", |
| "checkpoint_schedule": [ |
| 128 |
| ], |
| "context_parallel_size": 1, |
| "cosine_min_lr_ratio": 0.1, |
| "dataloader_num_workers": 1, |
| "dataloader_pin_memory": true, |
| "dataloader_prefetch_factor": 256, |
| "dataset_num_proc": 16, |
| "dataset_prepared_path": "/workspace/python4-aft-v2-20260813T154138Z-ordered_4ep/experiments/python4/aft_v2/runs/20260813T154138Z/arms/ordered_4ep/train/prepared", |
| "datasets": [ |
| { |
| "chat_template": "tokenizer_default", |
| "field_messages": "messages", |
| "message_property_mappings": { |
| "content": "content", |
| "role": "role" |
| }, |
| "path": "/workspace/python4-aft-v2-20260813T154138Z-ordered_4ep/experiments/python4/aft_v2/runs/20260813T154138Z/arms/ordered_4ep/aft_training.jsonl", |
| "trust_remote_code": false, |
| "type": "chat_template" |
| } |
| ], |
| "ddp": false, |
| "ddp_find_unused_parameters": true, |
| "device": "cuda:0", |
| "dion_rank_fraction": 1.0, |
| "dion_rank_multiple_of": 1, |
| "eaft_alpha": 1.0, |
| "eaft_k": 20, |
| "env_capabilities": { |
| "torch_version": "2.12.1" |
| }, |
| "eot_tokens": [ |
| "<end_of_turn>" |
| ], |
| "eval_batch_size": 4, |
| "eval_causal_lm_metrics": [ |
| "sacrebleu", |
| "comet", |
| "ter", |
| "chrf" |
| ], |
| "eval_max_new_tokens": 128, |
| "eval_table_size": 0, |
| "experimental_skip_move_to_device": true, |
| "fp16": false, |
| "generate_samples": false, |
| "generation_do_sample": true, |
| "generation_max_new_tokens": 50, |
| "generation_prompt_ratio": 0.5, |
| "generation_temperature": 0.7, |
| "gradient_accumulation_steps": 8, |
| "gradient_checkpointing": true, |
| "gradient_checkpointing_kwargs": { |
| "use_reentrant": false |
| }, |
| "include_tkps": true, |
| "is_multimodal": true, |
| "layer_offloading": false, |
| "learning_rate": 0.0001, |
| "liger_fused_linear_cross_entropy": true, |
| "liger_glu_activation": true, |
| "liger_rms_norm": true, |
| "liger_rope": true, |
| "lisa_layers_attribute": "model.layers", |
| "load_best_model_at_end": false, |
| "load_in_4bit": false, |
| "load_in_8bit": false, |
| "local_rank": 0, |
| "logging_steps": 1, |
| "lora_alpha": 128, |
| "lora_dropout": 0.0, |
| "lora_embedding_kernel": true, |
| "lora_mlp_kernel": true, |
| "lora_o_kernel": true, |
| "lora_qkv_kernel": true, |
| "lora_r": 64, |
| "lora_target_modules": [ |
| "model.language_model.layers.0.self_attn.q_proj", |
| "model.language_model.layers.0.self_attn.k_proj", |
| "model.language_model.layers.0.self_attn.v_proj", |
| "model.language_model.layers.0.self_attn.o_proj", |
| "model.language_model.layers.0.mlp.gate_proj", |
| "model.language_model.layers.0.mlp.up_proj", |
| "model.language_model.layers.0.mlp.down_proj", |
| "model.language_model.layers.1.self_attn.q_proj", |
| "model.language_model.layers.1.self_attn.k_proj", |
| "model.language_model.layers.1.self_attn.v_proj", |
| "model.language_model.layers.1.self_attn.o_proj", |
| "model.language_model.layers.1.mlp.gate_proj", |
| "model.language_model.layers.1.mlp.up_proj", |
| "model.language_model.layers.1.mlp.down_proj", |
| "model.language_model.layers.2.self_attn.q_proj", |
| "model.language_model.layers.2.self_attn.k_proj", |
| "model.language_model.layers.2.self_attn.v_proj", |
| "model.language_model.layers.2.self_attn.o_proj", |
| "model.language_model.layers.2.mlp.gate_proj", |
| "model.language_model.layers.2.mlp.up_proj", |
| "model.language_model.layers.2.mlp.down_proj", |
| "model.language_model.layers.3.self_attn.q_proj", |
| "model.language_model.layers.3.self_attn.k_proj", |
| "model.language_model.layers.3.self_attn.v_proj", |
| "model.language_model.layers.3.self_attn.o_proj", |
| "model.language_model.layers.3.mlp.gate_proj", |
| "model.language_model.layers.3.mlp.up_proj", |
| "model.language_model.layers.3.mlp.down_proj", |
| "model.language_model.layers.4.self_attn.q_proj", |
| "model.language_model.layers.4.self_attn.k_proj", |
| "model.language_model.layers.4.self_attn.v_proj", |
| "model.language_model.layers.4.self_attn.o_proj", |
| "model.language_model.layers.4.mlp.gate_proj", |
| "model.language_model.layers.4.mlp.up_proj", |
| "model.language_model.layers.4.mlp.down_proj", |
| "model.language_model.layers.5.self_attn.q_proj", |
| "model.language_model.layers.5.self_attn.k_proj", |
| "model.language_model.layers.5.self_attn.v_proj", |
| "model.language_model.layers.5.self_attn.o_proj", |
| "model.language_model.layers.5.mlp.gate_proj", |
| "model.language_model.layers.5.mlp.up_proj", |
| "model.language_model.layers.5.mlp.down_proj", |
| "model.language_model.layers.6.self_attn.q_proj", |
| "model.language_model.layers.6.self_attn.k_proj", |
| "model.language_model.layers.6.self_attn.v_proj", |
| "model.language_model.layers.6.self_attn.o_proj", |
| "model.language_model.layers.6.mlp.gate_proj", |
| "model.language_model.layers.6.mlp.up_proj", |
| "model.language_model.layers.6.mlp.down_proj", |
| "model.language_model.layers.7.self_attn.q_proj", |
| "model.language_model.layers.7.self_attn.k_proj", |
| "model.language_model.layers.7.self_attn.v_proj", |
| "model.language_model.layers.7.self_attn.o_proj", |
| "model.language_model.layers.7.mlp.gate_proj", |
| "model.language_model.layers.7.mlp.up_proj", |
| "model.language_model.layers.7.mlp.down_proj", |
| "model.language_model.layers.8.self_attn.q_proj", |
| "model.language_model.layers.8.self_attn.k_proj", |
| "model.language_model.layers.8.self_attn.v_proj", |
| "model.language_model.layers.8.self_attn.o_proj", |
| "model.language_model.layers.8.mlp.gate_proj", |
| "model.language_model.layers.8.mlp.up_proj", |
| "model.language_model.layers.8.mlp.down_proj", |
| "model.language_model.layers.9.self_attn.q_proj", |
| "model.language_model.layers.9.self_attn.k_proj", |
| "model.language_model.layers.9.self_attn.v_proj", |
| "model.language_model.layers.9.self_attn.o_proj", |
| "model.language_model.layers.9.mlp.gate_proj", |
| "model.language_model.layers.9.mlp.up_proj", |
| "model.language_model.layers.9.mlp.down_proj", |
| "model.language_model.layers.10.self_attn.q_proj", |
| "model.language_model.layers.10.self_attn.k_proj", |
| "model.language_model.layers.10.self_attn.v_proj", |
| "model.language_model.layers.10.self_attn.o_proj", |
| "model.language_model.layers.10.mlp.gate_proj", |
| "model.language_model.layers.10.mlp.up_proj", |
| "model.language_model.layers.10.mlp.down_proj", |
| "model.language_model.layers.11.self_attn.q_proj", |
| "model.language_model.layers.11.self_attn.k_proj", |
| "model.language_model.layers.11.self_attn.v_proj", |
| "model.language_model.layers.11.self_attn.o_proj", |
| "model.language_model.layers.11.mlp.gate_proj", |
| "model.language_model.layers.11.mlp.up_proj", |
| "model.language_model.layers.11.mlp.down_proj", |
| "model.language_model.layers.12.self_attn.q_proj", |
| "model.language_model.layers.12.self_attn.k_proj", |
| "model.language_model.layers.12.self_attn.v_proj", |
| "model.language_model.layers.12.self_attn.o_proj", |
| "model.language_model.layers.12.mlp.gate_proj", |
| "model.language_model.layers.12.mlp.up_proj", |
| "model.language_model.layers.12.mlp.down_proj", |
| "model.language_model.layers.13.self_attn.q_proj", |
| "model.language_model.layers.13.self_attn.k_proj", |
| "model.language_model.layers.13.self_attn.v_proj", |
| "model.language_model.layers.13.self_attn.o_proj", |
| "model.language_model.layers.13.mlp.gate_proj", |
| "model.language_model.layers.13.mlp.up_proj", |
| "model.language_model.layers.13.mlp.down_proj", |
| "model.language_model.layers.14.self_attn.q_proj", |
| "model.language_model.layers.14.self_attn.k_proj", |
| "model.language_model.layers.14.self_attn.v_proj", |
| "model.language_model.layers.14.self_attn.o_proj", |
| "model.language_model.layers.14.mlp.gate_proj", |
| "model.language_model.layers.14.mlp.up_proj", |
| "model.language_model.layers.14.mlp.down_proj", |
| "model.language_model.layers.15.self_attn.q_proj", |
| "model.language_model.layers.15.self_attn.k_proj", |
| "model.language_model.layers.15.self_attn.v_proj", |
| "model.language_model.layers.15.self_attn.o_proj", |
| "model.language_model.layers.15.mlp.gate_proj", |
| "model.language_model.layers.15.mlp.up_proj", |
| "model.language_model.layers.15.mlp.down_proj", |
| "model.language_model.layers.16.self_attn.q_proj", |
| "model.language_model.layers.16.self_attn.k_proj", |
| "model.language_model.layers.16.self_attn.v_proj", |
| "model.language_model.layers.16.self_attn.o_proj", |
| "model.language_model.layers.16.mlp.gate_proj", |
| "model.language_model.layers.16.mlp.up_proj", |
| "model.language_model.layers.16.mlp.down_proj", |
| "model.language_model.layers.17.self_attn.q_proj", |
| "model.language_model.layers.17.self_attn.k_proj", |
| "model.language_model.layers.17.self_attn.v_proj", |
| "model.language_model.layers.17.self_attn.o_proj", |
| "model.language_model.layers.17.mlp.gate_proj", |
| "model.language_model.layers.17.mlp.up_proj", |
| "model.language_model.layers.17.mlp.down_proj", |
| "model.language_model.layers.18.self_attn.q_proj", |
| "model.language_model.layers.18.self_attn.k_proj", |
| "model.language_model.layers.18.self_attn.v_proj", |
| "model.language_model.layers.18.self_attn.o_proj", |
| "model.language_model.layers.18.mlp.gate_proj", |
| "model.language_model.layers.18.mlp.up_proj", |
| "model.language_model.layers.18.mlp.down_proj", |
| "model.language_model.layers.19.self_attn.q_proj", |
| "model.language_model.layers.19.self_attn.k_proj", |
| "model.language_model.layers.19.self_attn.v_proj", |
| "model.language_model.layers.19.self_attn.o_proj", |
| "model.language_model.layers.19.mlp.gate_proj", |
| "model.language_model.layers.19.mlp.up_proj", |
| "model.language_model.layers.19.mlp.down_proj", |
| "model.language_model.layers.20.self_attn.q_proj", |
| "model.language_model.layers.20.self_attn.k_proj", |
| "model.language_model.layers.20.self_attn.v_proj", |
| "model.language_model.layers.20.self_attn.o_proj", |
| "model.language_model.layers.20.mlp.gate_proj", |
| "model.language_model.layers.20.mlp.up_proj", |
| "model.language_model.layers.20.mlp.down_proj", |
| "model.language_model.layers.21.self_attn.q_proj", |
| "model.language_model.layers.21.self_attn.k_proj", |
| "model.language_model.layers.21.self_attn.v_proj", |
| "model.language_model.layers.21.self_attn.o_proj", |
| "model.language_model.layers.21.mlp.gate_proj", |
| "model.language_model.layers.21.mlp.up_proj", |
| "model.language_model.layers.21.mlp.down_proj", |
| "model.language_model.layers.22.self_attn.q_proj", |
| "model.language_model.layers.22.self_attn.k_proj", |
| "model.language_model.layers.22.self_attn.v_proj", |
| "model.language_model.layers.22.self_attn.o_proj", |
| "model.language_model.layers.22.mlp.gate_proj", |
| "model.language_model.layers.22.mlp.up_proj", |
| "model.language_model.layers.22.mlp.down_proj", |
| "model.language_model.layers.23.self_attn.q_proj", |
| "model.language_model.layers.23.self_attn.k_proj", |
| "model.language_model.layers.23.self_attn.v_proj", |
| "model.language_model.layers.23.self_attn.o_proj", |
| "model.language_model.layers.23.mlp.gate_proj", |
| "model.language_model.layers.23.mlp.up_proj", |
| "model.language_model.layers.23.mlp.down_proj", |
| "model.language_model.layers.24.self_attn.q_proj", |
| "model.language_model.layers.24.self_attn.k_proj", |
| "model.language_model.layers.24.self_attn.v_proj", |
| "model.language_model.layers.24.self_attn.o_proj", |
| "model.language_model.layers.24.mlp.gate_proj", |
| "model.language_model.layers.24.mlp.up_proj", |
| "model.language_model.layers.24.mlp.down_proj", |
| "model.language_model.layers.25.self_attn.q_proj", |
| "model.language_model.layers.25.self_attn.k_proj", |
| "model.language_model.layers.25.self_attn.v_proj", |
| "model.language_model.layers.25.self_attn.o_proj", |
| "model.language_model.layers.25.mlp.gate_proj", |
| "model.language_model.layers.25.mlp.up_proj", |
| "model.language_model.layers.25.mlp.down_proj", |
| "model.language_model.layers.26.self_attn.q_proj", |
| "model.language_model.layers.26.self_attn.k_proj", |
| "model.language_model.layers.26.self_attn.v_proj", |
| "model.language_model.layers.26.self_attn.o_proj", |
| "model.language_model.layers.26.mlp.gate_proj", |
| "model.language_model.layers.26.mlp.up_proj", |
| "model.language_model.layers.26.mlp.down_proj", |
| "model.language_model.layers.27.self_attn.q_proj", |
| "model.language_model.layers.27.self_attn.k_proj", |
| "model.language_model.layers.27.self_attn.v_proj", |
| "model.language_model.layers.27.self_attn.o_proj", |
| "model.language_model.layers.27.mlp.gate_proj", |
| "model.language_model.layers.27.mlp.up_proj", |
| "model.language_model.layers.27.mlp.down_proj", |
| "model.language_model.layers.28.self_attn.q_proj", |
| "model.language_model.layers.28.self_attn.k_proj", |
| "model.language_model.layers.28.self_attn.v_proj", |
| "model.language_model.layers.28.self_attn.o_proj", |
| "model.language_model.layers.28.mlp.gate_proj", |
| "model.language_model.layers.28.mlp.up_proj", |
| "model.language_model.layers.28.mlp.down_proj", |
| "model.language_model.layers.29.self_attn.q_proj", |
| "model.language_model.layers.29.self_attn.k_proj", |
| "model.language_model.layers.29.self_attn.v_proj", |
| "model.language_model.layers.29.self_attn.o_proj", |
| "model.language_model.layers.29.mlp.gate_proj", |
| "model.language_model.layers.29.mlp.up_proj", |
| "model.language_model.layers.29.mlp.down_proj", |
| "model.language_model.layers.30.self_attn.q_proj", |
| "model.language_model.layers.30.self_attn.k_proj", |
| "model.language_model.layers.30.self_attn.v_proj", |
| "model.language_model.layers.30.self_attn.o_proj", |
| "model.language_model.layers.30.mlp.gate_proj", |
| "model.language_model.layers.30.mlp.up_proj", |
| "model.language_model.layers.30.mlp.down_proj", |
| "model.language_model.layers.31.self_attn.q_proj", |
| "model.language_model.layers.31.self_attn.k_proj", |
| "model.language_model.layers.31.self_attn.v_proj", |
| "model.language_model.layers.31.self_attn.o_proj", |
| "model.language_model.layers.31.mlp.gate_proj", |
| "model.language_model.layers.31.mlp.up_proj", |
| "model.language_model.layers.31.mlp.down_proj", |
| "model.language_model.layers.32.self_attn.q_proj", |
| "model.language_model.layers.32.self_attn.k_proj", |
| "model.language_model.layers.32.self_attn.v_proj", |
| "model.language_model.layers.32.self_attn.o_proj", |
| "model.language_model.layers.32.mlp.gate_proj", |
| "model.language_model.layers.32.mlp.up_proj", |
| "model.language_model.layers.32.mlp.down_proj", |
| "model.language_model.layers.33.self_attn.q_proj", |
| "model.language_model.layers.33.self_attn.k_proj", |
| "model.language_model.layers.33.self_attn.v_proj", |
| "model.language_model.layers.33.self_attn.o_proj", |
| "model.language_model.layers.33.mlp.gate_proj", |
| "model.language_model.layers.33.mlp.up_proj", |
| "model.language_model.layers.33.mlp.down_proj", |
| "model.language_model.layers.34.self_attn.q_proj", |
| "model.language_model.layers.34.self_attn.k_proj", |
| "model.language_model.layers.34.self_attn.v_proj", |
| "model.language_model.layers.34.self_attn.o_proj", |
| "model.language_model.layers.34.mlp.gate_proj", |
| "model.language_model.layers.34.mlp.up_proj", |
| "model.language_model.layers.34.mlp.down_proj", |
| "model.language_model.layers.35.self_attn.q_proj", |
| "model.language_model.layers.35.self_attn.k_proj", |
| "model.language_model.layers.35.self_attn.v_proj", |
| "model.language_model.layers.35.self_attn.o_proj", |
| "model.language_model.layers.35.mlp.gate_proj", |
| "model.language_model.layers.35.mlp.up_proj", |
| "model.language_model.layers.35.mlp.down_proj", |
| "model.language_model.layers.36.self_attn.q_proj", |
| "model.language_model.layers.36.self_attn.k_proj", |
| "model.language_model.layers.36.self_attn.v_proj", |
| "model.language_model.layers.36.self_attn.o_proj", |
| "model.language_model.layers.36.mlp.gate_proj", |
| "model.language_model.layers.36.mlp.up_proj", |
| "model.language_model.layers.36.mlp.down_proj", |
| "model.language_model.layers.37.self_attn.q_proj", |
| "model.language_model.layers.37.self_attn.k_proj", |
| "model.language_model.layers.37.self_attn.v_proj", |
| "model.language_model.layers.37.self_attn.o_proj", |
| "model.language_model.layers.37.mlp.gate_proj", |
| "model.language_model.layers.37.mlp.up_proj", |
| "model.language_model.layers.37.mlp.down_proj", |
| "model.language_model.layers.38.self_attn.q_proj", |
| "model.language_model.layers.38.self_attn.k_proj", |
| "model.language_model.layers.38.self_attn.v_proj", |
| "model.language_model.layers.38.self_attn.o_proj", |
| "model.language_model.layers.38.mlp.gate_proj", |
| "model.language_model.layers.38.mlp.up_proj", |
| "model.language_model.layers.38.mlp.down_proj", |
| "model.language_model.layers.39.self_attn.q_proj", |
| "model.language_model.layers.39.self_attn.k_proj", |
| "model.language_model.layers.39.self_attn.v_proj", |
| "model.language_model.layers.39.self_attn.o_proj", |
| "model.language_model.layers.39.mlp.gate_proj", |
| "model.language_model.layers.39.mlp.up_proj", |
| "model.language_model.layers.39.mlp.down_proj", |
| "model.language_model.layers.40.self_attn.q_proj", |
| "model.language_model.layers.40.self_attn.k_proj", |
| "model.language_model.layers.40.self_attn.v_proj", |
| "model.language_model.layers.40.self_attn.o_proj", |
| "model.language_model.layers.40.mlp.gate_proj", |
| "model.language_model.layers.40.mlp.up_proj", |
| "model.language_model.layers.40.mlp.down_proj", |
| "model.language_model.layers.41.self_attn.q_proj", |
| "model.language_model.layers.41.self_attn.k_proj", |
| "model.language_model.layers.41.self_attn.v_proj", |
| "model.language_model.layers.41.self_attn.o_proj", |
| "model.language_model.layers.41.mlp.gate_proj", |
| "model.language_model.layers.41.mlp.up_proj", |
| "model.language_model.layers.41.mlp.down_proj", |
| "model.language_model.layers.42.self_attn.q_proj", |
| "model.language_model.layers.42.self_attn.k_proj", |
| "model.language_model.layers.42.self_attn.v_proj", |
| "model.language_model.layers.42.self_attn.o_proj", |
| "model.language_model.layers.42.mlp.gate_proj", |
| "model.language_model.layers.42.mlp.up_proj", |
| "model.language_model.layers.42.mlp.down_proj", |
| "model.language_model.layers.43.self_attn.q_proj", |
| "model.language_model.layers.43.self_attn.k_proj", |
| "model.language_model.layers.43.self_attn.v_proj", |
| "model.language_model.layers.43.self_attn.o_proj", |
| "model.language_model.layers.43.mlp.gate_proj", |
| "model.language_model.layers.43.mlp.up_proj", |
| "model.language_model.layers.43.mlp.down_proj", |
| "model.language_model.layers.44.self_attn.q_proj", |
| "model.language_model.layers.44.self_attn.k_proj", |
| "model.language_model.layers.44.self_attn.v_proj", |
| "model.language_model.layers.44.self_attn.o_proj", |
| "model.language_model.layers.44.mlp.gate_proj", |
| "model.language_model.layers.44.mlp.up_proj", |
| "model.language_model.layers.44.mlp.down_proj", |
| "model.language_model.layers.45.self_attn.q_proj", |
| "model.language_model.layers.45.self_attn.k_proj", |
| "model.language_model.layers.45.self_attn.v_proj", |
| "model.language_model.layers.45.self_attn.o_proj", |
| "model.language_model.layers.45.mlp.gate_proj", |
| "model.language_model.layers.45.mlp.up_proj", |
| "model.language_model.layers.45.mlp.down_proj", |
| "model.language_model.layers.46.self_attn.q_proj", |
| "model.language_model.layers.46.self_attn.k_proj", |
| "model.language_model.layers.46.self_attn.v_proj", |
| "model.language_model.layers.46.self_attn.o_proj", |
| "model.language_model.layers.46.mlp.gate_proj", |
| "model.language_model.layers.46.mlp.up_proj", |
| "model.language_model.layers.46.mlp.down_proj", |
| "model.language_model.layers.47.self_attn.q_proj", |
| "model.language_model.layers.47.self_attn.k_proj", |
| "model.language_model.layers.47.self_attn.v_proj", |
| "model.language_model.layers.47.self_attn.o_proj", |
| "model.language_model.layers.47.mlp.gate_proj", |
| "model.language_model.layers.47.mlp.up_proj", |
| "model.language_model.layers.47.mlp.down_proj", |
| "model.language_model.layers.48.self_attn.q_proj", |
| "model.language_model.layers.48.self_attn.k_proj", |
| "model.language_model.layers.48.self_attn.v_proj", |
| "model.language_model.layers.48.self_attn.o_proj", |
| "model.language_model.layers.48.mlp.gate_proj", |
| "model.language_model.layers.48.mlp.up_proj", |
| "model.language_model.layers.48.mlp.down_proj", |
| "model.language_model.layers.49.self_attn.q_proj", |
| "model.language_model.layers.49.self_attn.k_proj", |
| "model.language_model.layers.49.self_attn.v_proj", |
| "model.language_model.layers.49.self_attn.o_proj", |
| "model.language_model.layers.49.mlp.gate_proj", |
| "model.language_model.layers.49.mlp.up_proj", |
| "model.language_model.layers.49.mlp.down_proj", |
| "model.language_model.layers.50.self_attn.q_proj", |
| "model.language_model.layers.50.self_attn.k_proj", |
| "model.language_model.layers.50.self_attn.v_proj", |
| "model.language_model.layers.50.self_attn.o_proj", |
| "model.language_model.layers.50.mlp.gate_proj", |
| "model.language_model.layers.50.mlp.up_proj", |
| "model.language_model.layers.50.mlp.down_proj", |
| "model.language_model.layers.51.self_attn.q_proj", |
| "model.language_model.layers.51.self_attn.k_proj", |
| "model.language_model.layers.51.self_attn.v_proj", |
| "model.language_model.layers.51.self_attn.o_proj", |
| "model.language_model.layers.51.mlp.gate_proj", |
| "model.language_model.layers.51.mlp.up_proj", |
| "model.language_model.layers.51.mlp.down_proj", |
| "model.language_model.layers.52.self_attn.q_proj", |
| "model.language_model.layers.52.self_attn.k_proj", |
| "model.language_model.layers.52.self_attn.v_proj", |
| "model.language_model.layers.52.self_attn.o_proj", |
| "model.language_model.layers.52.mlp.gate_proj", |
| "model.language_model.layers.52.mlp.up_proj", |
| "model.language_model.layers.52.mlp.down_proj", |
| "model.language_model.layers.53.self_attn.q_proj", |
| "model.language_model.layers.53.self_attn.k_proj", |
| "model.language_model.layers.53.self_attn.v_proj", |
| "model.language_model.layers.53.self_attn.o_proj", |
| "model.language_model.layers.53.mlp.gate_proj", |
| "model.language_model.layers.53.mlp.up_proj", |
| "model.language_model.layers.53.mlp.down_proj", |
| "model.language_model.layers.54.self_attn.q_proj", |
| "model.language_model.layers.54.self_attn.k_proj", |
| "model.language_model.layers.54.self_attn.v_proj", |
| "model.language_model.layers.54.self_attn.o_proj", |
| "model.language_model.layers.54.mlp.gate_proj", |
| "model.language_model.layers.54.mlp.up_proj", |
| "model.language_model.layers.54.mlp.down_proj", |
| "model.language_model.layers.55.self_attn.q_proj", |
| "model.language_model.layers.55.self_attn.k_proj", |
| "model.language_model.layers.55.self_attn.v_proj", |
| "model.language_model.layers.55.self_attn.o_proj", |
| "model.language_model.layers.55.mlp.gate_proj", |
| "model.language_model.layers.55.mlp.up_proj", |
| "model.language_model.layers.55.mlp.down_proj", |
| "model.language_model.layers.56.self_attn.q_proj", |
| "model.language_model.layers.56.self_attn.k_proj", |
| "model.language_model.layers.56.self_attn.v_proj", |
| "model.language_model.layers.56.self_attn.o_proj", |
| "model.language_model.layers.56.mlp.gate_proj", |
| "model.language_model.layers.56.mlp.up_proj", |
| "model.language_model.layers.56.mlp.down_proj", |
| "model.language_model.layers.57.self_attn.q_proj", |
| "model.language_model.layers.57.self_attn.k_proj", |
| "model.language_model.layers.57.self_attn.v_proj", |
| "model.language_model.layers.57.self_attn.o_proj", |
| "model.language_model.layers.57.mlp.gate_proj", |
| "model.language_model.layers.57.mlp.up_proj", |
| "model.language_model.layers.57.mlp.down_proj", |
| "model.language_model.layers.58.self_attn.q_proj", |
| "model.language_model.layers.58.self_attn.k_proj", |
| "model.language_model.layers.58.self_attn.v_proj", |
| "model.language_model.layers.58.self_attn.o_proj", |
| "model.language_model.layers.58.mlp.gate_proj", |
| "model.language_model.layers.58.mlp.up_proj", |
| "model.language_model.layers.58.mlp.down_proj", |
| "model.language_model.layers.59.self_attn.q_proj", |
| "model.language_model.layers.59.self_attn.k_proj", |
| "model.language_model.layers.59.self_attn.v_proj", |
| "model.language_model.layers.59.self_attn.o_proj", |
| "model.language_model.layers.59.mlp.gate_proj", |
| "model.language_model.layers.59.mlp.up_proj", |
| "model.language_model.layers.59.mlp.down_proj", |
| "model.language_model.layers.60.self_attn.q_proj", |
| "model.language_model.layers.60.self_attn.k_proj", |
| "model.language_model.layers.60.self_attn.v_proj", |
| "model.language_model.layers.60.self_attn.o_proj", |
| "model.language_model.layers.60.mlp.gate_proj", |
| "model.language_model.layers.60.mlp.up_proj", |
| "model.language_model.layers.60.mlp.down_proj", |
| "model.language_model.layers.61.self_attn.q_proj", |
| "model.language_model.layers.61.self_attn.k_proj", |
| "model.language_model.layers.61.self_attn.v_proj", |
| "model.language_model.layers.61.self_attn.o_proj", |
| "model.language_model.layers.61.mlp.gate_proj", |
| "model.language_model.layers.61.mlp.up_proj", |
| "model.language_model.layers.61.mlp.down_proj" |
| ], |
| "loraplus_lr_embedding": 1e-06, |
| "lr_scheduler": "cosine", |
| "max_grad_norm": 1.0, |
| "mean_resizing_embeddings": false, |
| "merge_method": "memory_efficient", |
| "micro_batch_size": 4, |
| "model_config_type": "gemma3", |
| "model_config_type_text": "gemma3_text", |
| "num_epochs": 4.0, |
| "num_generation_samples": 3, |
| "optimizer": "adamw_torch_fused", |
| "otel_metrics_host": "localhost", |
| "otel_metrics_port": 8000, |
| "output_dir": "/workspace/python4-aft-v2-20260813T154138Z-ordered_4ep/experiments/python4/aft_v2/runs/20260813T154138Z/arms/ordered_4ep/train/checkpoints", |
| "pad_to_sequence_len": false, |
| "plugins": [ |
| "axolotl.integrations.liger.LigerPlugin", |
| "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" |
| ], |
| "pretrain_multipack_attn": true, |
| "processor_config": "/workspace/python4-aft-v2-state/20260813T154138Z/ordered_4ep/parent/sdf_ordered/dolci_10m/end", |
| "profiler_steps_start": 0, |
| "qgalore_cos_threshold": 0.4, |
| "qgalore_gamma_proj": 2, |
| "qgalore_proj_bits": 4, |
| "qgalore_proj_group_size": 256, |
| "qgalore_proj_quant": true, |
| "qgalore_proj_type": "std", |
| "qgalore_queue_size": 5, |
| "qgalore_rank": 256, |
| "qgalore_scale": 0.25, |
| "qgalore_update_proj_gap": 200, |
| "qlora_sharded_model_loading": false, |
| "quantize_moe_experts": false, |
| "ray_num_workers": 1, |
| "relora_prune_method": "magnitude", |
| "resources_per_worker": { |
| "GPU": 1 |
| }, |
| "sample_packing": false, |
| "sample_packing_bin_size": 200, |
| "sample_packing_group_size": 100000, |
| "save_only_model": true, |
| "save_safetensors": true, |
| "save_strategy": "no", |
| "save_total_limit": 1, |
| "seed": 424242, |
| "sequence_len": 4096, |
| "shuffle_before_merging_datasets": false, |
| "shuffle_merged_datasets": true, |
| "skip_prepare_dataset": false, |
| "streaming_multipack_buffer_size": 10000, |
| "strict": false, |
| "tensor_parallel_size": 1, |
| "tf32": true, |
| "tiled_mlp_use_original_mlp": true, |
| "tokenizer_config": "/workspace/python4-aft-v2-state/20260813T154138Z/ordered_4ep/parent/sdf_ordered/dolci_10m/end", |
| "tokenizer_save_jinja_files": true, |
| "torch_dtype": "torch.bfloat16", |
| "train_on_inputs": false, |
| "trl": { |
| "async_prefetch": false, |
| "log_completions": false, |
| "mask_truncated_completions": false, |
| "ref_model_mixup_alpha": 0.9, |
| "ref_model_sync_steps": 64, |
| "replay_buffer_size": 0, |
| "replay_recompute_logps": true, |
| "reroll_max_groups": 1, |
| "reroll_start_fraction": 1.0, |
| "reward_num_workers": 1, |
| "scale_rewards": true, |
| "skip_zero_advantage_batches": true, |
| "sync_ref_model": false, |
| "use_data_producer": false, |
| "use_vllm": false, |
| "vllm_lora_sync": false, |
| "vllm_server_host": "0.0.0.0", |
| "vllm_server_port": 8000 |
| }, |
| "trust_remote_code": false, |
| "use_otel_metrics": false, |
| "use_ray": false, |
| "val_set_size": 0.0, |
| "vllm": { |
| "device": "auto", |
| "dtype": "auto", |
| "gpu_memory_utilization": 0.9, |
| "host": "0.0.0.0", |
| "port": 8000 |
| }, |
| "warmup_ratio": 0.05, |
| "weight_decay": 0.01, |
| "world_size": 1 |
| } |
| [2026-08-13 15:47:51,151] [DEBUG] [axolotl.loaders.utils.check_model_config:88] [PID:1520] Loaded image size: 896 from model config |
| [2026-08-13 15:47:52,524] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:311] [PID:1520] EOS: 106 / <end_of_turn> |
| [2026-08-13 15:47:52,525] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:312] [PID:1520] BOS: 2 / <bos> |
| [2026-08-13 15:47:52,525] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:313] [PID:1520] PAD: 0 / <pad> |
| [2026-08-13 15:47:52,525] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:314] [PID:1520] UNK: 3 / <unk> |
| [2026-08-13 15:47:52,526] [INFO] [axolotl.utils.data.shared.load_preprocessed_dataset:482] [PID:1520] Unable to find prepared dataset in /workspace/python4-aft-v2-20260813T154138Z-ordered_4ep/experiments/python4/aft_v2/runs/20260813T154138Z/arms/ordered_4ep/train/prepared/7389d5bae21c37f0bb9ae35059e4dbfb |
| [2026-08-13 15:47:52,526] [INFO] [axolotl.utils.data.sft._load_raw_datasets:320] [PID:1520] Loading raw datasets... |
| [2026-08-13 15:47:52,526] [WARNING] [axolotl.utils.data.sft._load_raw_datasets:322] [PID:1520] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. |
|
Generating train split: 0 examples [00:00, ? examples/s]
Generating train split: 1024 examples [00:00, 137729.84 examples/s] |
| [2026-08-13 15:47:52,687] [INFO] [axolotl.utils.data.wrappers.get_dataset_wrapper:87] [PID:1520] Loading dataset: /workspace/python4-aft-v2-20260813T154138Z-ordered_4ep/experiments/python4/aft_v2/runs/20260813T154138Z/arms/ordered_4ep/aft_training.jsonl with base_type: chat_template and prompt_style: None |
| [2026-08-13 15:47:52,695] [INFO] [axolotl.prompt_strategies.chat_template.__call__:1209] [PID:1520] Using chat template: |
| --- |
| {{ bos_token }} |
| {%- if messages[0]['role'] == 'system' -%} |
| {%- if messages[0]['content'] is string -%} |
| {%- set first_user_prefix = messages[0]['content'] + ' |
| |
| ' -%} |
| {%- else -%} |
| {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' |
| |
| ' -%} |
| {%- endif -%} |
| {%- set loop_messages = messages[1:] -%} |
| {%- else -%} |
| {%- set first_user_prefix = "" -%} |
| {%- set loop_messages = messages -%} |
| {%- endif -%} |
| {%- for message in loop_messages -%} |
| {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} |
| {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} |
| {%- endif -%} |
| {%- if (message['role'] == 'assistant') -%} |
| {%- set role = "model" -%} |
| {%- else -%} |
| {%- set role = message['role'] -%} |
| {%- endif -%} |
| {{ '<start_of_turn>' + role + ' |
| ' + (first_user_prefix if loop.first else "") }} |
| {%- if message['content'] is string -%} |
| {{ message['content'] | trim }} |
| {%- elif message['content'] is iterable -%} |
| {%- for item in message['content'] -%} |
| {%- if item['type'] == 'image' -%} |
| {{ '<start_of_image>' }} |
| {%- elif item['type'] == 'text' -%} |
| {{ item['text'] | trim }} |
| {%- endif -%} |
| {%- endfor -%} |
| {%- else -%} |
| {{ raise_exception("Invalid content type") }} |
| {%- endif -%} |
| {{ '<end_of_turn> |
| ' }} |
| {%- endfor -%} |
| {%- if add_generation_prompt -%} |
| {{'<start_of_turn>model |
| '}} |
| {%- endif -%} |
|
|
| --- |
|
Tokenizing Prompts (num_proc=16): 0%| | 0/1024 [00:00<?, ? examples/s]
Tokenizing Prompts (num_proc=16): 6%|▋ | 64/1024 [00:05<01:25, 11.22 examples/s]
Tokenizing Prompts (num_proc=16): 12%|█▎ | 128/1024 [00:09<01:06, 13.49 examples/s]
Tokenizing Prompts (num_proc=16): 19%|█▉ | 192/1024 [00:13<00:56, 14.76 examples/s]
Tokenizing Prompts (num_proc=16): 25%|██▌ | 256/1024 [00:17<00:49, 15.40 examples/s]
Tokenizing Prompts (num_proc=16): 31%|███▏ | 320/1024 [00:21<00:44, 15.69 examples/s]
Tokenizing Prompts (num_proc=16): 38%|███▊ | 384/1024 [00:25<00:40, 15.94 examples/s]
Tokenizing Prompts (num_proc=16): 44%|████▍ | 448/1024 [00:29<00:35, 16.08 examples/s]
Tokenizing Prompts (num_proc=16): 50%|█████ | 512/1024 [00:33<00:31, 16.17 examples/s]
Tokenizing Prompts (num_proc=16): 56%|█████▋ | 576/1024 [00:37<00:27, 16.24 examples/s]
Tokenizing Prompts (num_proc=16): 62%|██████▎ | 640/1024 [00:40<00:23, 16.25 examples/s]
Tokenizing Prompts (num_proc=16): 69%|██████▉ | 704/1024 [00:44<00:19, 16.25 examples/s]
Tokenizing Prompts (num_proc=16): 75%|███████▌ | 768/1024 [00:48<00:15, 16.37 examples/s]
Tokenizing Prompts (num_proc=16): 81%|████████▏ | 832/1024 [00:52<00:11, 16.40 examples/s]
Tokenizing Prompts (num_proc=16): 88%|████████▊ | 896/1024 [00:56<00:07, 16.39 examples/s]
Tokenizing Prompts (num_proc=16): 94%|█████████▍| 960/1024 [01:00<00:03, 16.44 examples/s]
Tokenizing Prompts (num_proc=16): 100%|██████████| 1024/1024 [01:04<00:00, 16.48 examples/s]
Tokenizing Prompts (num_proc=16): 100%|██████████| 1024/1024 [01:04<00:00, 15.86 examples/s] |
| [2026-08-13 15:49:13,064] [INFO] [axolotl.utils.data.utils._log_dataset_stats:212] [PID:1520] min_input_len: 255 |
| [2026-08-13 15:49:13,064] [INFO] [axolotl.utils.data.utils._log_dataset_stats:213] [PID:1520] max_input_len: 2091 |
|
Dropping Invalid Sequences (<None or >4096) (num_proc=16): 0%| | 0/1024 [00:00<?, ? examples/s]
Dropping Invalid Sequences (<None or >4096) (num_proc=16): 6%|▋ | 64/1024 [00:00<00:03, 280.62 examples/s]
Dropping Invalid Sequences (<None or >4096) (num_proc=16): 100%|██████████| 1024/1024 [00:00<00:00, 2826.34 examples/s] |
|
Saving the dataset (0/4 shards): 0%| | 0/1024 [00:00<?, ? examples/s]
Saving the dataset (0/4 shards): 25%|██▌ | 256/1024 [00:06<00:19, 39.69 examples/s]
Saving the dataset (1/4 shards): 25%|██▌ | 256/1024 [00:06<00:19, 39.69 examples/s]
Saving the dataset (2/4 shards): 50%|█████ | 512/1024 [00:06<00:12, 39.69 examples/s]
Saving the dataset (3/4 shards): 75%|███████▌ | 768/1024 [00:06<00:06, 39.69 examples/s]
Saving the dataset (4/4 shards): 100%|██████████| 1024/1024 [00:06<00:00, 39.69 examples/s]
Saving the dataset (4/4 shards): 100%|██████████| 1024/1024 [00:07<00:00, 135.25 examples/s] |
| [2026-08-13 15:49:21,122] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:420] [PID:1520] total_num_tokens: 654_917 |
| [2026-08-13 15:49:21,129] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:438] [PID:1520] `total_supervised_tokens: 236_779` |
| [2026-08-13 15:49:21,129] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:521] [PID:1520] total_num_steps: 128 |
| [2026-08-13 15:49:21,129] [INFO] [axolotl.utils.data.sft._prepare_standard_dataset:121] [PID:1520] Maximum number of steps set at 128 |
| [2026-08-13 15:49:21,257] [DEBUG] [axolotl.train.setup_model_and_tokenizer:70] [PID:1520] loading tokenizer... /workspace/python4-aft-v2-state/20260813T154138Z/ordered_4ep/parent/sdf_ordered/dolci_10m/end |
| [2026-08-13 15:49:22,761] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:311] [PID:1520] EOS: 106 / <end_of_turn> |
| [2026-08-13 15:49:22,761] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:312] [PID:1520] BOS: 2 / <bos> |
| [2026-08-13 15:49:22,761] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:313] [PID:1520] PAD: 0 / <pad> |
| [2026-08-13 15:49:22,761] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:314] [PID:1520] UNK: 3 / <unk> |
| [2026-08-13 15:49:24,556] [DEBUG] [axolotl.train.setup_model_and_tokenizer:81] [PID:1520] Loading model |
| [2026-08-13 15:49:24,562] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:75] [PID:1520] Patched OptimState8bit for torch.compile compatibility |
| [2026-08-13 15:49:24,563] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:122] [PID:1520] Patched OptimState4bit for torch.compile compatibility |
| [2026-08-13 15:49:24,563] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:154] [PID:1520] Patched OptimStateFp8 for torch.compile compatibility |
| [2026-08-13 15:49:24,568] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_evaluation_loop:94] [PID:1520] Patched Trainer.evaluation_loop with nanmean loss calculation |
| [2026-08-13 15:49:24,569] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_maybe_log_save_evaluate:148] [PID:1520] Patched Trainer._maybe_log_save_evaluate with nanmean loss calculation |
| [2026-08-13 15:49:24,624] [INFO] [axolotl.monkeypatch.attention.flash_attn_4.patch_flash_attn_4:52] [PID:1520] Flash Attention 4 is available for your GPU and offers faster training speeds. To enable: pip install flash-attn-4 |
| [2026-08-13 15:49:26,104] [INFO] [axolotl.monkeypatch.lora_kernels.patch_self_attn_lora:304] [PID:1520] Patched attention class with LoRA optims: Gemma3Attention |
| [2026-08-13 15:49:26,380] [INFO] [axolotl.integrations.liger.plugin.pre_model_load:117] [PID:1520] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} |
|
Loading weights: 0%| | 0/1248 [00:00<?, ?it/s]
Loading weights: 100%|██████████| 1248/1248 [00:00<00:00, 13538.59it/s] |
| [2026-08-13 15:49:31,366] [INFO] [axolotl.loaders.model._configure_embedding_dtypes:433] [PID:1520] Converting modules to torch.bfloat16 |
| [2026-08-13 15:49:33,673] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:1520] Memory usage after model load 0.000GB () |
| trainable params: 454,066,176 || all params: 27,886,472,816 || trainable%: 1.6283 |
| [2026-08-13 15:49:37,186] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:1520] after adapters 0.000GB () |
| [2026-08-13 15:49:48,706] [INFO] [axolotl.train.save_initial_configs:450] [PID:1520] Pre-saving adapter config to /workspace/python4-aft-v2-20260813T154138Z-ordered_4ep/experiments/python4/aft_v2/runs/20260813T154138Z/arms/ordered_4ep/train/checkpoints... |
| [2026-08-13 15:49:48,706] [INFO] [axolotl.train.save_initial_configs:454] [PID:1520] Pre-saving tokenizer to /workspace/python4-aft-v2-20260813T154138Z-ordered_4ep/experiments/python4/aft_v2/runs/20260813T154138Z/arms/ordered_4ep/train/checkpoints... |
| [2026-08-13 15:49:49,045] [INFO] [axolotl.train.save_initial_configs:459] [PID:1520] Pre-saving model config to /workspace/python4-aft-v2-20260813T154138Z-ordered_4ep/experiments/python4/aft_v2/runs/20260813T154138Z/arms/ordered_4ep/train/checkpoints... |
| [2026-08-13 15:49:49,049] [INFO] [axolotl.train.save_initial_configs:463] [PID:1520] Pre-saving processor to /workspace/python4-aft-v2-20260813T154138Z-ordered_4ep/experiments/python4/aft_v2/runs/20260813T154138Z/arms/ordered_4ep/train/checkpoints... |
| [2026-08-13 15:49:49,408] [INFO] [axolotl.train.execute_training:226] [PID:1520] Starting trainer... |
|
0%| | 0/128 [00:00<?, ?it/s]
1%| | 1/128 [00:13<29:30, 13.94s/it]
{'loss': '0.4785', 'grad_norm': '0.8015', 'learning_rate': '0', 'ppl': '1.614', 'memory/max_active (GiB)': '59.46', 'memory/max_allocated (GiB)': '59.46', 'memory/device_reserved (GiB)': '60.13', 'tokens/train_per_sec_per_gpu': '104.4', 'tokens/total': 28572, 'tokens/trainable': 7575, 'epoch': '0.03125'} |
|
1%| | 1/128 [00:13<29:30, 13.94s/it]
2%|▏ | 2/128 [00:24<25:09, 11.98s/it]
{'loss': '0.6591', 'grad_norm': '0.713', 'learning_rate': '1.667e-05', 'ppl': '1.933', 'memory/max_active (GiB)': '61.77', 'memory/max_allocated (GiB)': '61.77', 'memory/device_reserved (GiB)': '62.04', 'tokens/train_per_sec_per_gpu': '117.5', 'tokens/total': 56512, 'tokens/trainable': 15865, 'epoch': '0.0625'} |
|
2%|▏ | 2/128 [00:24<25:09, 11.98s/it]
2%|▏ | 3/128 [00:34<22:36, 10.85s/it]
{'loss': '0.4315', 'grad_norm': '0.7918', 'learning_rate': '3.333e-05', 'ppl': '1.54', 'memory/max_active (GiB)': '61.94', 'memory/max_allocated (GiB)': '61.94', 'memory/device_reserved (GiB)': '62.28', 'tokens/train_per_sec_per_gpu': '81.86', 'tokens/total': 82320, 'tokens/trainable': 23211, 'epoch': '0.09375'} |
|
2%|▏ | 3/128 [00:34<22:36, 10.85s/it]
3%|▎ | 4/128 [00:45<22:40, 10.97s/it]
{'loss': '0.3393', 'grad_norm': '0.6444', 'learning_rate': '5e-05', 'ppl': '1.404', 'memory/max_active (GiB)': '62.39', 'memory/max_allocated (GiB)': '62.39', 'memory/device_reserved (GiB)': '62.74', 'tokens/train_per_sec_per_gpu': '83.04', 'tokens/total': 112500, 'tokens/trainable': 30046, 'epoch': '0.125'} |
|
3%|▎ | 4/128 [00:45<22:40, 10.97s/it]
4%|▍ | 5/128 [00:55<22:19, 10.89s/it]
{'loss': '0.3506', 'grad_norm': '0.6094', 'learning_rate': '6.667e-05', 'ppl': '1.42', 'memory/max_active (GiB)': '65.24', 'memory/max_allocated (GiB)': '65.24', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '101.1', 'tokens/total': 142096, 'tokens/trainable': 37315, 'epoch': '0.1562'} |
|
4%|▍ | 5/128 [00:55<22:19, 10.89s/it]
5%|▍ | 6/128 [01:06<21:42, 10.67s/it]
{'loss': '0.4354', 'grad_norm': '0.6386', 'learning_rate': '8.333e-05', 'ppl': '1.546', 'memory/max_active (GiB)': '62.16', 'memory/max_allocated (GiB)': '62.16', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '96.81', 'tokens/total': 168284, 'tokens/trainable': 44600, 'epoch': '0.1875'} |
|
5%|▍ | 6/128 [01:06<21:42, 10.67s/it]
5%|▌ | 7/128 [01:16<21:20, 10.58s/it]
{'loss': '0.4155', 'grad_norm': '0.4957', 'learning_rate': '0.0001', 'ppl': '1.515', 'memory/max_active (GiB)': '61.52', 'memory/max_allocated (GiB)': '61.52', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '111.7', 'tokens/total': 196820, 'tokens/trainable': 52174, 'epoch': '0.2188'} |
|
5%|▌ | 7/128 [01:16<21:20, 10.58s/it]
6%|▋ | 8/128 [01:27<21:18, 10.66s/it]
{'loss': '0.3378', 'grad_norm': '0.3749', 'learning_rate': '9.999e-05', 'ppl': '1.402', 'memory/max_active (GiB)': '62.62', 'memory/max_allocated (GiB)': '62.62', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '112.6', 'tokens/total': 226052, 'tokens/trainable': 59475, 'epoch': '0.25'} |
|
6%|▋ | 8/128 [01:27<21:18, 10.66s/it]
7%|▋ | 9/128 [01:37<20:32, 10.35s/it]
{'loss': '0.3278', 'grad_norm': '0.3303', 'learning_rate': '9.994e-05', 'ppl': '1.388', 'memory/max_active (GiB)': '62.4', 'memory/max_allocated (GiB)': '62.4', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '68.07', 'tokens/total': 251360, 'tokens/trainable': 66221, 'epoch': '0.2812'} |
|
7%|▋ | 9/128 [01:37<20:32, 10.35s/it]
8%|▊ | 10/128 [01:48<20:55, 10.64s/it]
{'loss': '0.307', 'grad_norm': '0.3099', 'learning_rate': '9.987e-05', 'ppl': '1.359', 'memory/max_active (GiB)': '62.99', 'memory/max_allocated (GiB)': '62.99', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '102.3', 'tokens/total': 282048, 'tokens/trainable': 74909, 'epoch': '0.3125'} |
|
8%|▊ | 10/128 [01:48<20:55, 10.64s/it]
9%|▊ | 11/128 [01:57<19:39, 10.08s/it]
{'loss': '0.2443', 'grad_norm': '0.3531', 'learning_rate': '9.976e-05', 'ppl': '1.277', 'memory/max_active (GiB)': '61.38', 'memory/max_allocated (GiB)': '61.38', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '65.34', 'tokens/total': 304988, 'tokens/trainable': 80926, 'epoch': '0.3438'} |
|
9%|▊ | 11/128 [01:57<19:39, 10.08s/it]
9%|▉ | 12/128 [02:07<19:42, 10.19s/it]
{'loss': '0.3719', 'grad_norm': '0.3336', 'learning_rate': '9.963e-05', 'ppl': '1.45', 'memory/max_active (GiB)': '61.88', 'memory/max_allocated (GiB)': '61.88', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '80.91', 'tokens/total': 333108, 'tokens/trainable': 87992, 'epoch': '0.375'} |
|
9%|▉ | 12/128 [02:07<19:42, 10.19s/it]
10%|█ | 13/128 [02:17<19:09, 10.00s/it]
{'loss': '0.691', 'grad_norm': '1.967', 'learning_rate': '9.946e-05', 'ppl': '1.996', 'memory/max_active (GiB)': '60.88', 'memory/max_allocated (GiB)': '60.88', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '143.5', 'tokens/total': 356952, 'tokens/trainable': 96195, 'epoch': '0.4062'} |
|
10%|█ | 13/128 [02:17<19:09, 10.00s/it]
11%|█ | 14/128 [02:27<19:05, 10.05s/it]
{'loss': '0.318', 'grad_norm': '0.4091', 'learning_rate': '9.927e-05', 'ppl': '1.374', 'memory/max_active (GiB)': '61.4', 'memory/max_allocated (GiB)': '61.4', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '64.08', 'tokens/total': 384392, 'tokens/trainable': 103298, 'epoch': '0.4375'} |
|
11%|█ | 14/128 [02:27<19:05, 10.05s/it]
12%|█▏ | 15/128 [02:38<19:24, 10.31s/it]
{'loss': '0.3012', 'grad_norm': '0.4351', 'learning_rate': '9.905e-05', 'ppl': '1.351', 'memory/max_active (GiB)': '62.24', 'memory/max_allocated (GiB)': '62.24', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '58.51', 'tokens/total': 413280, 'tokens/trainable': 109986, 'epoch': '0.4688'} |
|
12%|█▏ | 15/128 [02:38<19:24, 10.31s/it]
12%|█▎ | 16/128 [02:49<19:38, 10.52s/it]
{'loss': '0.4406', 'grad_norm': '0.4501', 'learning_rate': '9.88e-05', 'ppl': '1.554', 'memory/max_active (GiB)': '62.53', 'memory/max_allocated (GiB)': '62.53', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '161.9', 'tokens/total': 443784, 'tokens/trainable': 118123, 'epoch': '0.5'} |
|
12%|█▎ | 16/128 [02:49<19:38, 10.52s/it]
13%|█▎ | 17/128 [03:00<19:47, 10.70s/it]
{'loss': '0.2982', 'grad_norm': '0.4526', 'learning_rate': '9.852e-05', 'ppl': '1.347', 'memory/max_active (GiB)': '62.91', 'memory/max_allocated (GiB)': '62.91', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '105.8', 'tokens/total': 473580, 'tokens/trainable': 126313, 'epoch': '0.5312'} |
|
13%|█▎ | 17/128 [03:00<19:47, 10.70s/it]
14%|█▍ | 18/128 [03:09<18:55, 10.32s/it]
{'loss': '0.263', 'grad_norm': '0.3519', 'learning_rate': '9.821e-05', 'ppl': '1.301', 'memory/max_active (GiB)': '61.13', 'memory/max_allocated (GiB)': '61.13', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '104.6', 'tokens/total': 498056, 'tokens/trainable': 133442, 'epoch': '0.5625'} |
|
14%|█▍ | 18/128 [03:09<18:55, 10.32s/it]
15%|█▍ | 19/128 [03:21<19:13, 10.58s/it]
{'loss': '0.2424', 'grad_norm': '0.3174', 'learning_rate': '9.787e-05', 'ppl': '1.274', 'memory/max_active (GiB)': '62.22', 'memory/max_allocated (GiB)': '62.22', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '53.87', 'tokens/total': 528120, 'tokens/trainable': 141989, 'epoch': '0.5938'} |
|
15%|█▍ | 19/128 [03:21<19:13, 10.58s/it]
16%|█▌ | 20/128 [03:31<19:01, 10.57s/it]
{'loss': '0.3873', 'grad_norm': '0.3435', 'learning_rate': '9.75e-05', 'ppl': '1.473', 'memory/max_active (GiB)': '62.31', 'memory/max_allocated (GiB)': '62.31', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '90.21', 'tokens/total': 557416, 'tokens/trainable': 149880, 'epoch': '0.625'} |
|
16%|█▌ | 20/128 [03:31<19:01, 10.57s/it]
16%|█▋ | 21/128 [03:41<18:35, 10.43s/it]
{'loss': '0.2882', 'grad_norm': '0.3157', 'learning_rate': '9.711e-05', 'ppl': '1.334', 'memory/max_active (GiB)': '61.34', 'memory/max_allocated (GiB)': '61.34', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '123.6', 'tokens/total': 583980, 'tokens/trainable': 158105, 'epoch': '0.6562'} |
|
16%|█▋ | 21/128 [03:41<18:35, 10.43s/it]
17%|█▋ | 22/128 [03:50<17:49, 10.09s/it]
{'loss': '0.2299', 'grad_norm': '0.3168', 'learning_rate': '9.668e-05', 'ppl': '1.259', 'memory/max_active (GiB)': '61.46', 'memory/max_allocated (GiB)': '61.46', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '114.4', 'tokens/total': 608384, 'tokens/trainable': 164670, 'epoch': '0.6875'} |
|
17%|█▋ | 22/128 [03:50<17:49, 10.09s/it]
18%|█▊ | 23/128 [04:00<17:36, 10.06s/it]
{'loss': '0.222', 'grad_norm': '0.2934', 'learning_rate': '9.623e-05', 'ppl': '1.249', 'memory/max_active (GiB)': '61.08', 'memory/max_allocated (GiB)': '61.08', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '114.4', 'tokens/total': 634132, 'tokens/trainable': 171415, 'epoch': '0.7188'} |
|
18%|█▊ | 23/128 [04:00<17:36, 10.06s/it]
19%|█▉ | 24/128 [04:10<16:58, 9.79s/it]
{'loss': '0.2172', 'grad_norm': '0.2828', 'learning_rate': '9.576e-05', 'ppl': '1.243', 'memory/max_active (GiB)': '61.04', 'memory/max_allocated (GiB)': '61.04', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '70.06', 'tokens/total': 657788, 'tokens/trainable': 177686, 'epoch': '0.75'} |
|
19%|█▉ | 24/128 [04:10<16:58, 9.79s/it]
20%|█▉ | 25/128 [04:20<17:01, 9.92s/it]
{'loss': '0.2539', 'grad_norm': '0.3245', 'learning_rate': '9.525e-05', 'ppl': '1.289', 'memory/max_active (GiB)': '61.38', 'memory/max_allocated (GiB)': '61.38', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '79.51', 'tokens/total': 684376, 'tokens/trainable': 185346, 'epoch': '0.7812'} |
|
20%|█▉ | 25/128 [04:20<17:01, 9.92s/it]
20%|██ | 26/128 [04:30<16:56, 9.97s/it]
{'loss': '0.3127', 'grad_norm': '0.34', 'learning_rate': '9.472e-05', 'ppl': '1.367', 'memory/max_active (GiB)': '61.86', 'memory/max_allocated (GiB)': '61.86', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '62.54', 'tokens/total': 711992, 'tokens/trainable': 192605, 'epoch': '0.8125'} |
|
20%|██ | 26/128 [04:30<16:56, 9.97s/it]
21%|██ | 27/128 [04:39<16:22, 9.73s/it]
{'loss': '0.264', 'grad_norm': '0.2998', 'learning_rate': '9.416e-05', 'ppl': '1.302', 'memory/max_active (GiB)': '61.57', 'memory/max_allocated (GiB)': '61.57', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '66.83', 'tokens/total': 734816, 'tokens/trainable': 198978, 'epoch': '0.8438'} |
|
21%|██ | 27/128 [04:39<16:22, 9.73s/it]
22%|██▏ | 28/128 [04:50<16:44, 10.05s/it]
{'loss': '0.2732', 'grad_norm': '0.2578', 'learning_rate': '9.358e-05', 'ppl': '1.314', 'memory/max_active (GiB)': '62.85', 'memory/max_allocated (GiB)': '62.85', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '123.3', 'tokens/total': 763996, 'tokens/trainable': 207138, 'epoch': '0.875'} |
|
22%|██▏ | 28/128 [04:50<16:44, 10.05s/it]
23%|██▎ | 29/128 [05:00<16:42, 10.13s/it]
{'loss': '0.479', 'grad_norm': '0.3087', 'learning_rate': '9.297e-05', 'ppl': '1.614', 'memory/max_active (GiB)': '63', 'memory/max_allocated (GiB)': '63', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '64.69', 'tokens/total': 792560, 'tokens/trainable': 214833, 'epoch': '0.9062'} |
|
23%|██▎ | 29/128 [05:00<16:42, 10.13s/it]
23%|██▎ | 30/128 [05:10<16:35, 10.16s/it]
{'loss': '0.285', 'grad_norm': '0.315', 'learning_rate': '9.234e-05', 'ppl': '1.33', 'memory/max_active (GiB)': '61.8', 'memory/max_allocated (GiB)': '61.8', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '85.13', 'tokens/total': 818900, 'tokens/trainable': 222291, 'epoch': '0.9375'} |
|
23%|██▎ | 30/128 [05:10<16:35, 10.16s/it]
24%|██▍ | 31/128 [05:20<16:19, 10.10s/it]
{'loss': '0.3202', 'grad_norm': '0.3232', 'learning_rate': '9.168e-05', 'ppl': '1.377', 'memory/max_active (GiB)': '61.47', 'memory/max_allocated (GiB)': '61.47', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '63.35', 'tokens/total': 845156, 'tokens/trainable': 229659, 'epoch': '0.9688'} |
|
24%|██▍ | 31/128 [05:20<16:19, 10.10s/it]
25%|██▌ | 32/128 [05:30<16:08, 10.09s/it]
{'loss': '0.3317', 'grad_norm': '0.3311', 'learning_rate': '9.099e-05', 'ppl': '1.393', 'memory/max_active (GiB)': '62.03', 'memory/max_allocated (GiB)': '62.03', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '83.8', 'tokens/total': 870564, 'tokens/trainable': 236779, 'epoch': '1'} |
|
25%|██▌ | 32/128 [05:30<16:08, 10.09s/it]
26%|██▌ | 33/128 [05:40<15:47, 9.98s/it]
{'loss': '0.3938', 'grad_norm': '0.3177', 'learning_rate': '9.029e-05', 'ppl': '1.483', 'memory/max_active (GiB)': '61.23', 'memory/max_allocated (GiB)': '61.23', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '133.1', 'tokens/total': 894852, 'tokens/trainable': 244968, 'epoch': '1.031'} |
|
26%|██▌ | 33/128 [05:40<15:47, 9.98s/it]
27%|██▋ | 34/128 [05:51<16:05, 10.27s/it]
{'loss': '0.172', 'grad_norm': '0.2575', 'learning_rate': '8.955e-05', 'ppl': '1.188', 'memory/max_active (GiB)': '61.85', 'memory/max_allocated (GiB)': '61.85', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '99.81', 'tokens/total': 923880, 'tokens/trainable': 251510, 'epoch': '1.062'} |
|
27%|██▋ | 34/128 [05:51<16:05, 10.27s/it]
27%|██▋ | 35/128 [06:01<15:48, 10.20s/it]
{'loss': '0.1943', 'grad_norm': '0.2605', 'learning_rate': '8.88e-05', 'ppl': '1.215', 'memory/max_active (GiB)': '61.83', 'memory/max_allocated (GiB)': '61.83', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '123.9', 'tokens/total': 951396, 'tokens/trainable': 258730, 'epoch': '1.094'} |
|
27%|██▋ | 35/128 [06:01<15:48, 10.20s/it]
28%|██▊ | 36/128 [06:11<15:33, 10.14s/it]
{'loss': '0.2659', 'grad_norm': '0.3572', 'learning_rate': '8.802e-05', 'ppl': '1.305', 'memory/max_active (GiB)': '62.39', 'memory/max_allocated (GiB)': '62.39', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '63.59', 'tokens/total': 977128, 'tokens/trainable': 265887, 'epoch': '1.125'} |
|
28%|██▊ | 36/128 [06:11<15:33, 10.14s/it]
29%|██▉ | 37/128 [06:21<15:16, 10.07s/it]
{'loss': '0.2185', 'grad_norm': '0.2978', 'learning_rate': '8.723e-05', 'ppl': '1.244', 'memory/max_active (GiB)': '61.45', 'memory/max_allocated (GiB)': '61.45', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '63.88', 'tokens/total': 1003672, 'tokens/trainable': 272560, 'epoch': '1.156'} |
|
29%|██▉ | 37/128 [06:21<15:16, 10.07s/it]
30%|██▉ | 38/128 [06:32<15:21, 10.24s/it]
{'loss': '0.1984', 'grad_norm': '0.3082', 'learning_rate': '8.641e-05', 'ppl': '1.219', 'memory/max_active (GiB)': '61.95', 'memory/max_allocated (GiB)': '61.95', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '115.6', 'tokens/total': 1031564, 'tokens/trainable': 280802, 'epoch': '1.188'} |
|
30%|██▉ | 38/128 [06:32<15:21, 10.24s/it]
30%|███ | 39/128 [06:42<15:03, 10.16s/it]
{'loss': '0.2541', 'grad_norm': '0.3415', 'learning_rate': '8.557e-05', 'ppl': '1.289', 'memory/max_active (GiB)': '62.17', 'memory/max_allocated (GiB)': '62.17', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '135.7', 'tokens/total': 1058404, 'tokens/trainable': 288020, 'epoch': '1.219'} |
|
30%|███ | 39/128 [06:42<15:03, 10.16s/it]
31%|███▏ | 40/128 [06:52<14:49, 10.11s/it]
{'loss': '0.2177', 'grad_norm': '0.3485', 'learning_rate': '8.471e-05', 'ppl': '1.243', 'memory/max_active (GiB)': '62.44', 'memory/max_allocated (GiB)': '62.44', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '90.66', 'tokens/total': 1084208, 'tokens/trainable': 295082, 'epoch': '1.25'} |
|
31%|███▏ | 40/128 [06:52<14:49, 10.11s/it]
32%|███▏ | 41/128 [07:01<14:30, 10.00s/it]
{'loss': '0.1545', 'grad_norm': '0.3964', 'learning_rate': '8.383e-05', 'ppl': '1.167', 'memory/max_active (GiB)': '61.51', 'memory/max_allocated (GiB)': '61.51', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '87.02', 'tokens/total': 1110428, 'tokens/trainable': 301185, 'epoch': '1.281'} |
|
32%|███▏ | 41/128 [07:01<14:30, 10.00s/it]
33%|███▎ | 42/128 [07:12<14:47, 10.32s/it]
{'loss': '0.2033', 'grad_norm': '0.3585', 'learning_rate': '8.293e-05', 'ppl': '1.225', 'memory/max_active (GiB)': '62.62', 'memory/max_allocated (GiB)': '62.62', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '91.46', 'tokens/total': 1139844, 'tokens/trainable': 309987, 'epoch': '1.312'} |
|
33%|███▎ | 42/128 [07:12<14:47, 10.32s/it]
34%|███▎ | 43/128 [07:22<14:27, 10.20s/it]
{'loss': '0.144', 'grad_norm': '0.318', 'learning_rate': '8.201e-05', 'ppl': '1.155', 'memory/max_active (GiB)': '61.87', 'memory/max_allocated (GiB)': '61.87', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '70.07', 'tokens/total': 1167068, 'tokens/trainable': 317133, 'epoch': '1.344'} |
|
34%|███▎ | 43/128 [07:22<14:27, 10.20s/it]
34%|███▍ | 44/128 [07:33<14:14, 10.18s/it]
{'loss': '0.192', 'grad_norm': '0.606', 'learning_rate': '8.107e-05', 'ppl': '1.212', 'memory/max_active (GiB)': '61.36', 'memory/max_allocated (GiB)': '61.36', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '93.07', 'tokens/total': 1192996, 'tokens/trainable': 324270, 'epoch': '1.375'} |
|
34%|███▍ | 44/128 [07:33<14:14, 10.18s/it]
35%|███▌ | 45/128 [07:43<14:01, 10.14s/it]
{'loss': '0.2864', 'grad_norm': '0.5503', 'learning_rate': '8.012e-05', 'ppl': '1.332', 'memory/max_active (GiB)': '61.62', 'memory/max_allocated (GiB)': '61.62', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '100.1', 'tokens/total': 1218580, 'tokens/trainable': 331679, 'epoch': '1.406'} |
|
35%|███▌ | 45/128 [07:43<14:01, 10.14s/it]
36%|███▌ | 46/128 [07:53<13:52, 10.16s/it]
{'loss': '0.2007', 'grad_norm': '0.3526', 'learning_rate': '7.915e-05', 'ppl': '1.222', 'memory/max_active (GiB)': '62.4', 'memory/max_allocated (GiB)': '62.4', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '80.41', 'tokens/total': 1246588, 'tokens/trainable': 339009, 'epoch': '1.438'} |
|
36%|███▌ | 46/128 [07:53<13:52, 10.16s/it]
37%|███▋ | 47/128 [08:02<13:21, 9.89s/it]
{'loss': '0.2435', 'grad_norm': '0.3343', 'learning_rate': '7.817e-05', 'ppl': '1.276', 'memory/max_active (GiB)': '60.99', 'memory/max_allocated (GiB)': '60.99', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '113.4', 'tokens/total': 1269360, 'tokens/trainable': 346896, 'epoch': '1.469'} |
|
37%|███▋ | 47/128 [08:02<13:21, 9.89s/it]
38%|███▊ | 48/128 [08:12<13:01, 9.77s/it]
{'loss': '0.2149', 'grad_norm': '0.3195', 'learning_rate': '7.716e-05', 'ppl': '1.24', 'memory/max_active (GiB)': '61.49', 'memory/max_allocated (GiB)': '61.49', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '69.49', 'tokens/total': 1293848, 'tokens/trainable': 354504, 'epoch': '1.5'} |
|
38%|███▊ | 48/128 [08:12<13:01, 9.77s/it]
38%|███▊ | 49/128 [08:23<13:38, 10.36s/it]
{'loss': '0.1807', 'grad_norm': '0.4253', 'learning_rate': '7.615e-05', 'ppl': '1.198', 'memory/max_active (GiB)': '62.99', 'memory/max_allocated (GiB)': '62.99', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '109.6', 'tokens/total': 1326248, 'tokens/trainable': 363672, 'epoch': '1.531'} |
|
38%|███▊ | 49/128 [08:23<13:38, 10.36s/it]
39%|███▉ | 50/128 [08:33<13:07, 10.10s/it]
{'loss': '0.2421', 'grad_norm': '0.3941', 'learning_rate': '7.512e-05', 'ppl': '1.274', 'memory/max_active (GiB)': '61.29', 'memory/max_allocated (GiB)': '61.29', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '71.64', 'tokens/total': 1351092, 'tokens/trainable': 370812, 'epoch': '1.562'} |
|
39%|███▉ | 50/128 [08:33<13:07, 10.10s/it]
40%|███▉ | 51/128 [08:42<12:43, 9.91s/it]
{'loss': '0.1525', 'grad_norm': '0.3378', 'learning_rate': '7.408e-05', 'ppl': '1.165', 'memory/max_active (GiB)': '61.36', 'memory/max_allocated (GiB)': '61.36', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '111.8', 'tokens/total': 1375188, 'tokens/trainable': 378084, 'epoch': '1.594'} |
|
40%|███▉ | 51/128 [08:42<12:43, 9.91s/it]
41%|████ | 52/128 [08:52<12:41, 10.01s/it]
{'loss': '0.2014', 'grad_norm': '0.3111', 'learning_rate': '7.302e-05', 'ppl': '1.223', 'memory/max_active (GiB)': '62.85', 'memory/max_allocated (GiB)': '62.85', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '82.22', 'tokens/total': 1403484, 'tokens/trainable': 385450, 'epoch': '1.625'} |
|
41%|████ | 52/128 [08:52<12:41, 10.01s/it]
41%|████▏ | 53/128 [09:03<12:38, 10.12s/it]
{'loss': '0.31', 'grad_norm': '0.4114', 'learning_rate': '7.195e-05', 'ppl': '1.363', 'memory/max_active (GiB)': '61.45', 'memory/max_allocated (GiB)': '61.45', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '77.62', 'tokens/total': 1430556, 'tokens/trainable': 393200, 'epoch': '1.656'} |
|
41%|████▏ | 53/128 [09:03<12:38, 10.12s/it]
42%|████▏ | 54/128 [09:15<13:05, 10.61s/it]
{'loss': '0.1443', 'grad_norm': '0.3084', 'learning_rate': '7.087e-05', 'ppl': '1.155', 'memory/max_active (GiB)': '62.53', 'memory/max_allocated (GiB)': '62.53', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '120', 'tokens/total': 1464160, 'tokens/trainable': 401245, 'epoch': '1.688'} |
|
42%|████▏ | 54/128 [09:15<13:05, 10.61s/it]
43%|████▎ | 55/128 [09:25<12:52, 10.59s/it]
{'loss': '0.1483', 'grad_norm': '0.3515', 'learning_rate': '6.978e-05', 'ppl': '1.16', 'memory/max_active (GiB)': '61.87', 'memory/max_allocated (GiB)': '61.87', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '93.84', 'tokens/total': 1492720, 'tokens/trainable': 407748, 'epoch': '1.719'} |
|
43%|████▎ | 55/128 [09:25<12:52, 10.59s/it]
44%|████▍ | 56/128 [09:35<12:25, 10.35s/it]
{'loss': '0.2101', 'grad_norm': '0.3444', 'learning_rate': '6.869e-05', 'ppl': '1.234', 'memory/max_active (GiB)': '62.91', 'memory/max_allocated (GiB)': '62.91', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '107.8', 'tokens/total': 1519496, 'tokens/trainable': 414529, 'epoch': '1.75'} |
|
44%|████▍ | 56/128 [09:35<12:25, 10.35s/it]
45%|████▍ | 57/128 [09:45<12:09, 10.27s/it]
{'loss': '0.1894', 'grad_norm': '0.3912', 'learning_rate': '6.758e-05', 'ppl': '1.209', 'memory/max_active (GiB)': '61.92', 'memory/max_allocated (GiB)': '61.92', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '107.6', 'tokens/total': 1545740, 'tokens/trainable': 421282, 'epoch': '1.781'} |
|
45%|████▍ | 57/128 [09:45<12:09, 10.27s/it]
45%|████▌ | 58/128 [09:55<11:52, 10.18s/it]
{'loss': '0.2563', 'grad_norm': '0.7853', 'learning_rate': '6.646e-05', 'ppl': '1.292', 'memory/max_active (GiB)': '61.42', 'memory/max_allocated (GiB)': '61.42', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '65.97', 'tokens/total': 1573476, 'tokens/trainable': 429253, 'epoch': '1.812'} |
|
45%|████▌ | 58/128 [09:55<11:52, 10.18s/it]
46%|████▌ | 59/128 [10:05<11:45, 10.23s/it]
{'loss': '0.2409', 'grad_norm': '0.3774', 'learning_rate': '6.534e-05', 'ppl': '1.272', 'memory/max_active (GiB)': '62.38', 'memory/max_allocated (GiB)': '62.38', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '77.65', 'tokens/total': 1600756, 'tokens/trainable': 437205, 'epoch': '1.844'} |
|
46%|████▌ | 59/128 [10:05<11:45, 10.23s/it]
47%|████▋ | 60/128 [10:16<11:50, 10.44s/it]
{'loss': '0.2497', 'grad_norm': '0.3271', 'learning_rate': '6.42e-05', 'ppl': '1.284', 'memory/max_active (GiB)': '62.21', 'memory/max_allocated (GiB)': '62.21', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '137.2', 'tokens/total': 1631348, 'tokens/trainable': 445596, 'epoch': '1.875'} |
|
47%|████▋ | 60/128 [10:16<11:50, 10.44s/it]
48%|████▊ | 61/128 [10:27<11:41, 10.47s/it]
{'loss': '0.1445', 'grad_norm': '0.4376', 'learning_rate': '6.307e-05', 'ppl': '1.155', 'memory/max_active (GiB)': '62.84', 'memory/max_allocated (GiB)': '62.84', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '52.45', 'tokens/total': 1659460, 'tokens/trainable': 451831, 'epoch': '1.906'} |
|
48%|████▊ | 61/128 [10:27<11:41, 10.47s/it]
48%|████▊ | 62/128 [10:36<11:05, 10.09s/it]
{'loss': '0.2149', 'grad_norm': '0.6615', 'learning_rate': '6.193e-05', 'ppl': '1.24', 'memory/max_active (GiB)': '61.16', 'memory/max_allocated (GiB)': '61.16', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '74.18', 'tokens/total': 1683816, 'tokens/trainable': 458074, 'epoch': '1.938'} |
|
48%|████▊ | 62/128 [10:36<11:05, 10.09s/it]
49%|████▉ | 63/128 [10:46<10:51, 10.02s/it]
{'loss': '0.327', 'grad_norm': '0.365', 'learning_rate': '6.078e-05', 'ppl': '1.387', 'memory/max_active (GiB)': '61.46', 'memory/max_allocated (GiB)': '61.46', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '105.1', 'tokens/total': 1709084, 'tokens/trainable': 466953, 'epoch': '1.969'} |
|
49%|████▉ | 63/128 [10:46<10:51, 10.02s/it]
50%|█████ | 64/128 [10:57<11:05, 10.39s/it]
{'loss': '0.1505', 'grad_norm': '0.3441', 'learning_rate': '5.963e-05', 'ppl': '1.162', 'memory/max_active (GiB)': '65.23', 'memory/max_allocated (GiB)': '65.23', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '61.81', 'tokens/total': 1739072, 'tokens/trainable': 473558, 'epoch': '2'} |
|
50%|█████ | 64/128 [10:57<11:05, 10.39s/it]
51%|█████ | 65/128 [11:07<10:42, 10.20s/it]
{'loss': '0.1314', 'grad_norm': '0.4317', 'learning_rate': '5.847e-05', 'ppl': '1.14', 'memory/max_active (GiB)': '61.13', 'memory/max_allocated (GiB)': '61.13', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '109.9', 'tokens/total': 1763640, 'tokens/trainable': 481217, 'epoch': '2.031'} |
|
51%|█████ | 65/128 [11:07<10:42, 10.20s/it]
52%|█████▏ | 66/128 [11:18<10:49, 10.48s/it]
{'loss': '0.08951', 'grad_norm': '0.2726', 'learning_rate': '5.732e-05', 'ppl': '1.094', 'memory/max_active (GiB)': '62.2', 'memory/max_allocated (GiB)': '62.2', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '87.51', 'tokens/total': 1793396, 'tokens/trainable': 487839, 'epoch': '2.062'} |
|
52%|█████▏ | 66/128 [11:18<10:49, 10.48s/it]
52%|█████▏ | 67/128 [11:28<10:22, 10.21s/it]
{'loss': '0.1076', 'grad_norm': '0.3386', 'learning_rate': '5.616e-05', 'ppl': '1.114', 'memory/max_active (GiB)': '61.78', 'memory/max_allocated (GiB)': '61.78', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '100.8', 'tokens/total': 1818448, 'tokens/trainable': 493902, 'epoch': '2.094'} |
|
52%|█████▏ | 67/128 [11:28<10:22, 10.21s/it]
53%|█████▎ | 68/128 [11:40<10:46, 10.78s/it]
{'loss': '0.1024', 'grad_norm': '0.2761', 'learning_rate': '5.5e-05', 'ppl': '1.108', 'memory/max_active (GiB)': '65.25', 'memory/max_allocated (GiB)': '65.25', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '71.26', 'tokens/total': 1852036, 'tokens/trainable': 502094, 'epoch': '2.125'} |
|
53%|█████▎ | 68/128 [11:40<10:46, 10.78s/it]
54%|█████▍ | 69/128 [11:50<10:30, 10.69s/it]
{'loss': '0.1359', 'grad_norm': '0.366', 'learning_rate': '5.384e-05', 'ppl': '1.146', 'memory/max_active (GiB)': '61.71', 'memory/max_allocated (GiB)': '61.71', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '96.79', 'tokens/total': 1880656, 'tokens/trainable': 510593, 'epoch': '2.156'} |
|
54%|█████▍ | 69/128 [11:50<10:30, 10.69s/it]
55%|█████▍ | 70/128 [12:00<10:03, 10.41s/it]
{'loss': '0.1639', 'grad_norm': '0.5084', 'learning_rate': '5.268e-05', 'ppl': '1.178', 'memory/max_active (GiB)': '61.28', 'memory/max_allocated (GiB)': '61.28', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '105.3', 'tokens/total': 1905028, 'tokens/trainable': 518250, 'epoch': '2.188'} |
|
55%|█████▍ | 70/128 [12:00<10:03, 10.41s/it]
55%|█████▌ | 71/128 [12:09<09:34, 10.08s/it]
{'loss': '0.2041', 'grad_norm': '0.6129', 'learning_rate': '5.153e-05', 'ppl': '1.226', 'memory/max_active (GiB)': '61.1', 'memory/max_allocated (GiB)': '61.1', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '113.4', 'tokens/total': 1929140, 'tokens/trainable': 526478, 'epoch': '2.219'} |
|
55%|█████▌ | 71/128 [12:09<09:34, 10.08s/it]
56%|█████▋ | 72/128 [12:19<09:17, 9.96s/it]
{'loss': '0.102', 'grad_norm': '0.5932', 'learning_rate': '5.037e-05', 'ppl': '1.107', 'memory/max_active (GiB)': '60.92', 'memory/max_allocated (GiB)': '60.92', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '59.3', 'tokens/total': 1953896, 'tokens/trainable': 533140, 'epoch': '2.25'} |
|
56%|█████▋ | 72/128 [12:19<09:17, 9.96s/it]
57%|█████▋ | 73/128 [12:29<09:07, 9.96s/it]
{'loss': '0.09013', 'grad_norm': '0.4671', 'learning_rate': '4.922e-05', 'ppl': '1.094', 'memory/max_active (GiB)': '60.96', 'memory/max_allocated (GiB)': '60.96', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '75.17', 'tokens/total': 1980148, 'tokens/trainable': 539469, 'epoch': '2.281'} |
|
57%|█████▋ | 73/128 [12:29<09:07, 9.96s/it]
58%|█████▊ | 74/128 [12:39<08:55, 9.92s/it]
{'loss': '0.1521', 'grad_norm': '0.5446', 'learning_rate': '4.807e-05', 'ppl': '1.164', 'memory/max_active (GiB)': '61.59', 'memory/max_allocated (GiB)': '61.59', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '93.58', 'tokens/total': 2004292, 'tokens/trainable': 546977, 'epoch': '2.312'} |
|
58%|█████▊ | 74/128 [12:39<08:55, 9.92s/it]
59%|█████▊ | 75/128 [12:49<08:52, 10.05s/it]
{'loss': '0.1002', 'grad_norm': '0.4876', 'learning_rate': '4.693e-05', 'ppl': '1.105', 'memory/max_active (GiB)': '62.3', 'memory/max_allocated (GiB)': '62.3', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '75.12', 'tokens/total': 2032520, 'tokens/trainable': 553216, 'epoch': '2.344'} |
|
59%|█████▊ | 75/128 [12:49<08:52, 10.05s/it]
59%|█████▉ | 76/128 [13:01<09:09, 10.57s/it]
{'loss': '0.0931', 'grad_norm': '0.4191', 'learning_rate': '4.58e-05', 'ppl': '1.098', 'memory/max_active (GiB)': '62.53', 'memory/max_allocated (GiB)': '62.53', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '102.5', 'tokens/total': 2064644, 'tokens/trainable': 560800, 'epoch': '2.375'} |
|
59%|█████▉ | 76/128 [13:01<09:09, 10.57s/it]
60%|██████ | 77/128 [13:11<08:52, 10.44s/it]
{'loss': '0.1779', 'grad_norm': '0.6629', 'learning_rate': '4.466e-05', 'ppl': '1.195', 'memory/max_active (GiB)': '61.61', 'memory/max_allocated (GiB)': '61.61', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '80.68', 'tokens/total': 2090812, 'tokens/trainable': 567665, 'epoch': '2.406'} |
|
60%|██████ | 77/128 [13:11<08:52, 10.44s/it]
61%|██████ | 78/128 [13:22<08:49, 10.59s/it]
{'loss': '0.1292', 'grad_norm': '0.4356', 'learning_rate': '4.354e-05', 'ppl': '1.138', 'memory/max_active (GiB)': '63.01', 'memory/max_allocated (GiB)': '63.01', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '100.7', 'tokens/total': 2121540, 'tokens/trainable': 575399, 'epoch': '2.438'} |
|
61%|██████ | 78/128 [13:22<08:49, 10.59s/it]
62%|██████▏ | 79/128 [13:32<08:28, 10.38s/it]
{'loss': '0.1408', 'grad_norm': '0.5232', 'learning_rate': '4.242e-05', 'ppl': '1.151', 'memory/max_active (GiB)': '61.05', 'memory/max_allocated (GiB)': '61.05', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '137', 'tokens/total': 2146788, 'tokens/trainable': 582816, 'epoch': '2.469'} |
|
62%|██████▏ | 79/128 [13:32<08:28, 10.38s/it]
62%|██████▎ | 80/128 [13:42<08:10, 10.21s/it]
{'loss': '0.1153', 'grad_norm': '0.4014', 'learning_rate': '4.131e-05', 'ppl': '1.122', 'memory/max_active (GiB)': '62.4', 'memory/max_allocated (GiB)': '62.4', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '100.5', 'tokens/total': 2173012, 'tokens/trainable': 590228, 'epoch': '2.5'} |
|
62%|██████▎ | 80/128 [13:42<08:10, 10.21s/it]
63%|██████▎ | 81/128 [13:53<08:11, 10.46s/it]
{'loss': '0.1185', 'grad_norm': '0.384', 'learning_rate': '4.022e-05', 'ppl': '1.126', 'memory/max_active (GiB)': '62.43', 'memory/max_allocated (GiB)': '62.43', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '99.75', 'tokens/total': 2202820, 'tokens/trainable': 597439, 'epoch': '2.531'} |
|
63%|██████▎ | 81/128 [13:53<08:11, 10.46s/it]
64%|██████▍ | 82/128 [14:03<07:56, 10.36s/it]
{'loss': '0.2111', 'grad_norm': '0.3701', 'learning_rate': '3.913e-05', 'ppl': '1.235', 'memory/max_active (GiB)': '61.94', 'memory/max_allocated (GiB)': '61.94', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '110', 'tokens/total': 2230636, 'tokens/trainable': 606167, 'epoch': '2.562'} |
|
64%|██████▍ | 82/128 [14:03<07:56, 10.36s/it]
65%|██████▍ | 83/128 [14:13<07:42, 10.29s/it]
{'loss': '0.08004', 'grad_norm': '0.4051', 'learning_rate': '3.805e-05', 'ppl': '1.083', 'memory/max_active (GiB)': '61.33', 'memory/max_allocated (GiB)': '61.33', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '85.89', 'tokens/total': 2257360, 'tokens/trainable': 613034, 'epoch': '2.594'} |
|
65%|██████▍ | 83/128 [14:13<07:42, 10.29s/it]
66%|██████▌ | 84/128 [14:23<07:26, 10.15s/it]
{'loss': '0.1214', 'grad_norm': '0.3691', 'learning_rate': '3.698e-05', 'ppl': '1.129', 'memory/max_active (GiB)': '62.98', 'memory/max_allocated (GiB)': '62.98', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '80.88', 'tokens/total': 2284004, 'tokens/trainable': 620094, 'epoch': '2.625'} |
|
66%|██████▌ | 84/128 [14:23<07:26, 10.15s/it]
66%|██████▋ | 85/128 [14:32<07:10, 10.01s/it]
{'loss': '0.09699', 'grad_norm': '0.4162', 'learning_rate': '3.592e-05', 'ppl': '1.102', 'memory/max_active (GiB)': '61.19', 'memory/max_allocated (GiB)': '61.19', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '121.9', 'tokens/total': 2308528, 'tokens/trainable': 626836, 'epoch': '2.656'} |
|
66%|██████▋ | 85/128 [14:32<07:10, 10.01s/it]
67%|██████▋ | 86/128 [14:44<07:15, 10.38s/it]
{'loss': '0.1604', 'grad_norm': '0.4854', 'learning_rate': '3.488e-05', 'ppl': '1.174', 'memory/max_active (GiB)': '62.85', 'memory/max_allocated (GiB)': '62.85', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '101', 'tokens/total': 2340360, 'tokens/trainable': 635364, 'epoch': '2.688'} |
|
67%|██████▋ | 86/128 [14:44<07:15, 10.38s/it]
68%|██████▊ | 87/128 [14:53<06:53, 10.09s/it]
{'loss': '0.09827', 'grad_norm': '0.3785', 'learning_rate': '3.385e-05', 'ppl': '1.103', 'memory/max_active (GiB)': '61.33', 'memory/max_allocated (GiB)': '61.33', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '122.7', 'tokens/total': 2364116, 'tokens/trainable': 643111, 'epoch': '2.719'} |
|
68%|██████▊ | 87/128 [14:53<06:53, 10.09s/it]
69%|██████▉ | 88/128 [15:03<06:42, 10.07s/it]
{'loss': '0.09678', 'grad_norm': '0.5675', 'learning_rate': '3.284e-05', 'ppl': '1.102', 'memory/max_active (GiB)': '61.39', 'memory/max_allocated (GiB)': '61.39', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '117', 'tokens/total': 2390936, 'tokens/trainable': 650643, 'epoch': '2.75'} |
|
69%|██████▉ | 88/128 [15:03<06:42, 10.07s/it]
70%|██████▉ | 89/128 [15:14<06:44, 10.37s/it]
{'loss': '0.1146', 'grad_norm': '0.4922', 'learning_rate': '3.183e-05', 'ppl': '1.121', 'memory/max_active (GiB)': '62.93', 'memory/max_allocated (GiB)': '62.93', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '114.8', 'tokens/total': 2421276, 'tokens/trainable': 658072, 'epoch': '2.781'} |
|
70%|██████▉ | 89/128 [15:14<06:44, 10.37s/it]
70%|███████ | 90/128 [15:25<06:43, 10.62s/it]
{'loss': '0.08986', 'grad_norm': '0.3809', 'learning_rate': '3.085e-05', 'ppl': '1.094', 'memory/max_active (GiB)': '62.61', 'memory/max_allocated (GiB)': '62.61', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '69.94', 'tokens/total': 2452932, 'tokens/trainable': 665766, 'epoch': '2.812'} |
|
70%|███████ | 90/128 [15:25<06:43, 10.62s/it]
71%|███████ | 91/128 [15:35<06:26, 10.46s/it]
{'loss': '0.08168', 'grad_norm': '0.4902', 'learning_rate': '2.988e-05', 'ppl': '1.085', 'memory/max_active (GiB)': '61.91', 'memory/max_allocated (GiB)': '61.91', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '69.12', 'tokens/total': 2479584, 'tokens/trainable': 672013, 'epoch': '2.844'} |
|
71%|███████ | 91/128 [15:35<06:26, 10.46s/it]
72%|███████▏ | 92/128 [15:46<06:18, 10.51s/it]
{'loss': '0.09928', 'grad_norm': '0.4507', 'learning_rate': '2.893e-05', 'ppl': '1.104', 'memory/max_active (GiB)': '62.03', 'memory/max_allocated (GiB)': '62.03', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '80.58', 'tokens/total': 2506956, 'tokens/trainable': 679699, 'epoch': '2.875'} |
|
72%|███████▏ | 92/128 [15:46<06:18, 10.51s/it]
73%|███████▎ | 93/128 [15:57<06:15, 10.72s/it]
{'loss': '0.1059', 'grad_norm': '1.102', 'learning_rate': '2.799e-05', 'ppl': '1.112', 'memory/max_active (GiB)': '62.39', 'memory/max_allocated (GiB)': '62.39', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '83.58', 'tokens/total': 2538408, 'tokens/trainable': 686967, 'epoch': '2.906'} |
|
73%|███████▎ | 93/128 [15:57<06:15, 10.72s/it]
73%|███████▎ | 94/128 [16:08<06:02, 10.66s/it]
{'loss': '0.1205', 'grad_norm': '0.4804', 'learning_rate': '2.707e-05', 'ppl': '1.128', 'memory/max_active (GiB)': '62.03', 'memory/max_allocated (GiB)': '62.03', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '63.02', 'tokens/total': 2566044, 'tokens/trainable': 695138, 'epoch': '2.938'} |
|
73%|███████▎ | 94/128 [16:08<06:02, 10.66s/it]
74%|███████▍ | 95/128 [16:17<05:38, 10.26s/it]
{'loss': '0.09688', 'grad_norm': '0.47', 'learning_rate': '2.617e-05', 'ppl': '1.102', 'memory/max_active (GiB)': '61.47', 'memory/max_allocated (GiB)': '61.47', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '146.9', 'tokens/total': 2591040, 'tokens/trainable': 702275, 'epoch': '2.969'} |
|
74%|███████▍ | 95/128 [16:17<05:38, 10.26s/it]
75%|███████▌ | 96/128 [16:27<05:26, 10.19s/it]
{'loss': '0.1305', 'grad_norm': '0.4172', 'learning_rate': '2.529e-05', 'ppl': '1.139', 'memory/max_active (GiB)': '61.55', 'memory/max_allocated (GiB)': '61.55', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '111.9', 'tokens/total': 2616776, 'tokens/trainable': 710337, 'epoch': '3'} |
|
75%|███████▌ | 96/128 [16:27<05:26, 10.19s/it]
76%|███████▌ | 97/128 [16:38<05:21, 10.37s/it]
{'loss': '0.07586', 'grad_norm': '0.3154', 'learning_rate': '2.443e-05', 'ppl': '1.079', 'memory/max_active (GiB)': '62.44', 'memory/max_allocated (GiB)': '62.44', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '122.5', 'tokens/total': 2645272, 'tokens/trainable': 717607, 'epoch': '3.031'} |
|
76%|███████▌ | 97/128 [16:38<05:21, 10.37s/it]
77%|███████▋ | 98/128 [16:48<05:09, 10.32s/it]
{'loss': '0.08307', 'grad_norm': '0.3687', 'learning_rate': '2.359e-05', 'ppl': '1.087', 'memory/max_active (GiB)': '61.86', 'memory/max_allocated (GiB)': '61.86', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '106.3', 'tokens/total': 2671656, 'tokens/trainable': 725080, 'epoch': '3.062'} |
|
77%|███████▋ | 98/128 [16:48<05:09, 10.32s/it]
77%|███████▋ | 99/128 [16:57<04:48, 9.96s/it]
{'loss': '0.06115', 'grad_norm': '0.3489', 'learning_rate': '2.277e-05', 'ppl': '1.063', 'memory/max_active (GiB)': '61.11', 'memory/max_allocated (GiB)': '61.11', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '106.3', 'tokens/total': 2695780, 'tokens/trainable': 732499, 'epoch': '3.094'} |
|
77%|███████▋ | 99/128 [16:57<04:48, 9.96s/it]
78%|███████▊ | 100/128 [17:07<04:35, 9.84s/it]
{'loss': '0.06124', 'grad_norm': '0.336', 'learning_rate': '2.198e-05', 'ppl': '1.063', 'memory/max_active (GiB)': '61.7', 'memory/max_allocated (GiB)': '61.7', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '83.07', 'tokens/total': 2720188, 'tokens/trainable': 739851, 'epoch': '3.125'} |
|
78%|███████▊ | 100/128 [17:07<04:35, 9.84s/it]
79%|███████▉ | 101/128 [17:18<04:36, 10.24s/it]
{'loss': '0.05766', 'grad_norm': '0.3022', 'learning_rate': '2.12e-05', 'ppl': '1.059', 'memory/max_active (GiB)': '62.84', 'memory/max_allocated (GiB)': '62.84', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '60.75', 'tokens/total': 2751896, 'tokens/trainable': 747545, 'epoch': '3.156'} |
|
79%|███████▉ | 101/128 [17:18<04:36, 10.24s/it]
80%|███████▉ | 102/128 [17:28<04:25, 10.20s/it]
{'loss': '0.09628', 'grad_norm': '0.4117', 'learning_rate': '2.045e-05', 'ppl': '1.101', 'memory/max_active (GiB)': '62.03', 'memory/max_allocated (GiB)': '62.03', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '68.79', 'tokens/total': 2777824, 'tokens/trainable': 754408, 'epoch': '3.188'} |
|
80%|███████▉ | 102/128 [17:28<04:25, 10.20s/it]
80%|████████ | 103/128 [17:38<04:14, 10.17s/it]
{'loss': '0.04058', 'grad_norm': '0.4061', 'learning_rate': '1.971e-05', 'ppl': '1.041', 'memory/max_active (GiB)': '61.42', 'memory/max_allocated (GiB)': '61.42', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '82.72', 'tokens/total': 2805324, 'tokens/trainable': 760831, 'epoch': '3.219'} |
|
80%|████████ | 103/128 [17:38<04:14, 10.17s/it]
81%|████████▏ | 104/128 [17:48<04:04, 10.17s/it]
{'loss': '0.0566', 'grad_norm': '0.4065', 'learning_rate': '1.901e-05', 'ppl': '1.058', 'memory/max_active (GiB)': '61.83', 'memory/max_allocated (GiB)': '61.83', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '131.1', 'tokens/total': 2832068, 'tokens/trainable': 769191, 'epoch': '3.25'} |
|
81%|████████▏ | 104/128 [17:48<04:04, 10.17s/it]
82%|████████▏ | 105/128 [17:58<03:52, 10.12s/it]
{'loss': '0.0492', 'grad_norm': '0.4664', 'learning_rate': '1.832e-05', 'ppl': '1.05', 'memory/max_active (GiB)': '61.4', 'memory/max_allocated (GiB)': '61.4', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '68.69', 'tokens/total': 2859496, 'tokens/trainable': 776065, 'epoch': '3.281'} |
|
82%|████████▏ | 105/128 [17:58<03:52, 10.12s/it]
83%|████████▎ | 106/128 [18:09<03:46, 10.31s/it]
{'loss': '0.06405', 'grad_norm': '0.5535', 'learning_rate': '1.766e-05', 'ppl': '1.066', 'memory/max_active (GiB)': '62.4', 'memory/max_allocated (GiB)': '62.4', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '88.05', 'tokens/total': 2888168, 'tokens/trainable': 783919, 'epoch': '3.312'} |
|
83%|████████▎ | 106/128 [18:09<03:46, 10.31s/it]
84%|████████▎ | 107/128 [18:21<03:43, 10.62s/it]
{'loss': '0.04397', 'grad_norm': '0.5903', 'learning_rate': '1.703e-05', 'ppl': '1.045', 'memory/max_active (GiB)': '63', 'memory/max_allocated (GiB)': '63', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '85.43', 'tokens/total': 2919108, 'tokens/trainable': 791343, 'epoch': '3.344'} |
|
84%|████████▎ | 107/128 [18:21<03:43, 10.62s/it]
84%|████████▍ | 108/128 [18:31<03:29, 10.48s/it]
{'loss': '0.06368', 'grad_norm': '0.4812', 'learning_rate': '1.642e-05', 'ppl': '1.066', 'memory/max_active (GiB)': '62.2', 'memory/max_allocated (GiB)': '62.2', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '104.9', 'tokens/total': 2946728, 'tokens/trainable': 799664, 'epoch': '3.375'} |
|
84%|████████▍ | 108/128 [18:31<03:29, 10.48s/it]
85%|████████▌ | 109/128 [18:40<03:14, 10.24s/it]
{'loss': '0.05835', 'grad_norm': '0.5516', 'learning_rate': '1.584e-05', 'ppl': '1.06', 'memory/max_active (GiB)': '62.24', 'memory/max_allocated (GiB)': '62.24', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '128.3', 'tokens/total': 2971256, 'tokens/trainable': 808020, 'epoch': '3.406'} |
|
85%|████████▌ | 109/128 [18:40<03:14, 10.24s/it]
86%|████████▌ | 110/128 [18:50<03:02, 10.12s/it]
{'loss': '0.07576', 'grad_norm': '0.4191', 'learning_rate': '1.528e-05', 'ppl': '1.079', 'memory/max_active (GiB)': '62.98', 'memory/max_allocated (GiB)': '62.98', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '135.8', 'tokens/total': 2998056, 'tokens/trainable': 816376, 'epoch': '3.438'} |
|
86%|████████▌ | 110/128 [18:50<03:02, 10.12s/it]
87%|████████▋ | 111/128 [19:00<02:52, 10.14s/it]
{'loss': '0.06401', 'grad_norm': '0.625', 'learning_rate': '1.475e-05', 'ppl': '1.066', 'memory/max_active (GiB)': '61.46', 'memory/max_allocated (GiB)': '61.46', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '95.85', 'tokens/total': 3025116, 'tokens/trainable': 824339, 'epoch': '3.469'} |
|
87%|████████▋ | 111/128 [19:00<02:52, 10.14s/it]
88%|████████▊ | 112/128 [19:10<02:41, 10.06s/it]
{'loss': '0.06046', 'grad_norm': '0.596', 'learning_rate': '1.424e-05', 'ppl': '1.062', 'memory/max_active (GiB)': '61.77', 'memory/max_allocated (GiB)': '61.77', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '86.53', 'tokens/total': 3051800, 'tokens/trainable': 831162, 'epoch': '3.5'} |
|
88%|████████▊ | 112/128 [19:10<02:41, 10.06s/it]
88%|████████▊ | 113/128 [19:20<02:30, 10.06s/it]
{'loss': '0.04294', 'grad_norm': '0.6525', 'learning_rate': '1.377e-05', 'ppl': '1.044', 'memory/max_active (GiB)': '61.58', 'memory/max_allocated (GiB)': '61.58', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '86.74', 'tokens/total': 3077536, 'tokens/trainable': 837060, 'epoch': '3.531'} |
|
88%|████████▊ | 113/128 [19:20<02:30, 10.06s/it]
89%|████████▉ | 114/128 [19:30<02:17, 9.83s/it]
{'loss': '0.06368', 'grad_norm': '0.5504', 'learning_rate': '1.332e-05', 'ppl': '1.066', 'memory/max_active (GiB)': '61.22', 'memory/max_allocated (GiB)': '61.22', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '145.6', 'tokens/total': 3101764, 'tokens/trainable': 845319, 'epoch': '3.562'} |
|
89%|████████▉ | 114/128 [19:30<02:17, 9.83s/it]
90%|████████▉ | 115/128 [19:39<02:06, 9.77s/it]
{'loss': '0.05019', 'grad_norm': '0.5034', 'learning_rate': '1.289e-05', 'ppl': '1.051', 'memory/max_active (GiB)': '61.61', 'memory/max_allocated (GiB)': '61.61', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '58.59', 'tokens/total': 3125720, 'tokens/trainable': 852314, 'epoch': '3.594'} |
|
90%|████████▉ | 115/128 [19:39<02:06, 9.77s/it]
91%|█████████ | 116/128 [19:49<01:57, 9.76s/it]
{'loss': '0.04404', 'grad_norm': '0.6253', 'learning_rate': '1.25e-05', 'ppl': '1.045', 'memory/max_active (GiB)': '61.85', 'memory/max_allocated (GiB)': '61.85', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '81.13', 'tokens/total': 3152324, 'tokens/trainable': 858307, 'epoch': '3.625'} |
|
91%|█████████ | 116/128 [19:49<01:57, 9.76s/it]
91%|█████████▏| 117/128 [19:59<01:49, 9.91s/it]
{'loss': '0.04827', 'grad_norm': '0.3911', 'learning_rate': '1.213e-05', 'ppl': '1.049', 'memory/max_active (GiB)': '62.94', 'memory/max_allocated (GiB)': '62.94', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '144.9', 'tokens/total': 3179492, 'tokens/trainable': 865234, 'epoch': '3.656'} |
|
91%|█████████▏| 117/128 [19:59<01:49, 9.91s/it]
92%|█████████▏| 118/128 [20:09<01:37, 9.79s/it]
{'loss': '0.04272', 'grad_norm': '0.515', 'learning_rate': '1.179e-05', 'ppl': '1.044', 'memory/max_active (GiB)': '61.06', 'memory/max_allocated (GiB)': '61.06', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '77.38', 'tokens/total': 3204652, 'tokens/trainable': 871968, 'epoch': '3.688'} |
|
92%|█████████▏| 118/128 [20:09<01:37, 9.79s/it]
93%|█████████▎| 119/128 [20:20<01:30, 10.10s/it]
{'loss': '0.06366', 'grad_norm': '0.5084', 'learning_rate': '1.148e-05', 'ppl': '1.066', 'memory/max_active (GiB)': '62.16', 'memory/max_allocated (GiB)': '62.16', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '55.33', 'tokens/total': 3233940, 'tokens/trainable': 878865, 'epoch': '3.719'} |
|
93%|█████████▎| 119/128 [20:20<01:30, 10.10s/it]
94%|█████████▍| 120/128 [20:30<01:21, 10.19s/it]
{'loss': '0.04247', 'grad_norm': '0.4849', 'learning_rate': '1.12e-05', 'ppl': '1.043', 'memory/max_active (GiB)': '61.76', 'memory/max_allocated (GiB)': '61.76', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '104.8', 'tokens/total': 3262500, 'tokens/trainable': 886214, 'epoch': '3.75'} |
|
94%|█████████▍| 120/128 [20:30<01:21, 10.19s/it]
95%|█████████▍| 121/128 [20:41<01:12, 10.42s/it]
{'loss': '0.04529', 'grad_norm': '0.493', 'learning_rate': '1.095e-05', 'ppl': '1.046', 'memory/max_active (GiB)': '62.61', 'memory/max_allocated (GiB)': '62.61', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '118.7', 'tokens/total': 3292288, 'tokens/trainable': 893566, 'epoch': '3.781'} |
|
95%|█████████▍| 121/128 [20:41<01:12, 10.42s/it]
95%|█████████▌| 122/128 [20:51<01:01, 10.22s/it]
{'loss': '0.08027', 'grad_norm': '0.4469', 'learning_rate': '1.073e-05', 'ppl': '1.084', 'memory/max_active (GiB)': '61.17', 'memory/max_allocated (GiB)': '61.17', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '72.95', 'tokens/total': 3318256, 'tokens/trainable': 901937, 'epoch': '3.812'} |
|
95%|█████████▌| 122/128 [20:51<01:01, 10.22s/it]
96%|█████████▌| 123/128 [21:01<00:50, 10.13s/it]
{'loss': '0.04132', 'grad_norm': '0.6718', 'learning_rate': '1.054e-05', 'ppl': '1.042', 'memory/max_active (GiB)': '62.91', 'memory/max_allocated (GiB)': '62.91', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '53.38', 'tokens/total': 3344436, 'tokens/trainable': 909008, 'epoch': '3.844'} |
|
96%|█████████▌| 123/128 [21:01<00:50, 10.13s/it]
97%|█████████▋| 124/128 [21:11<00:40, 10.14s/it]
{'loss': '0.03903', 'grad_norm': '0.5238', 'learning_rate': '1.037e-05', 'ppl': '1.04', 'memory/max_active (GiB)': '62.39', 'memory/max_allocated (GiB)': '62.39', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '67.4', 'tokens/total': 3371716, 'tokens/trainable': 915519, 'epoch': '3.875'} |
|
97%|█████████▋| 124/128 [21:11<00:40, 10.14s/it]
98%|█████████▊| 125/128 [21:23<00:32, 10.67s/it]
{'loss': '0.04167', 'grad_norm': '0.3464', 'learning_rate': '1.024e-05', 'ppl': '1.043', 'memory/max_active (GiB)': '62.85', 'memory/max_allocated (GiB)': '62.85', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '152.3', 'tokens/total': 3404300, 'tokens/trainable': 924818, 'epoch': '3.906'} |
|
98%|█████████▊| 125/128 [21:23<00:32, 10.67s/it]
98%|█████████▊| 126/128 [21:33<00:21, 10.65s/it]
{'loss': '0.04875', 'grad_norm': '0.4427', 'learning_rate': '1.013e-05', 'ppl': '1.05', 'memory/max_active (GiB)': '62.31', 'memory/max_allocated (GiB)': '62.31', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '111.6', 'tokens/total': 3434008, 'tokens/trainable': 931637, 'epoch': '3.938'} |
|
98%|█████████▊| 126/128 [21:33<00:21, 10.65s/it]
99%|█████████▉| 127/128 [21:44<00:10, 10.78s/it]
{'loss': '0.05882', 'grad_norm': '0.3732', 'learning_rate': '1.006e-05', 'ppl': '1.061', 'memory/max_active (GiB)': '65.22', 'memory/max_allocated (GiB)': '65.22', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '66.28', 'tokens/total': 3464128, 'tokens/trainable': 938304, 'epoch': '3.969'} |
|
99%|█████████▉| 127/128 [21:44<00:10, 10.78s/it]
100%|██████████| 128/128 [21:55<00:00, 10.69s/it]
{'loss': '0.08117', 'grad_norm': '0.4681', 'learning_rate': '1.001e-05', 'ppl': '1.085', 'memory/max_active (GiB)': '62.03', 'memory/max_allocated (GiB)': '62.03', 'memory/device_reserved (GiB)': '65.73', 'tokens/train_per_sec_per_gpu': '116', 'tokens/total': 3492372, 'tokens/trainable': 947116, 'epoch': '4'} |
|
100%|██████████| 128/128 [21:55<00:00, 10.69s/it][2026-08-13 16:11:45,217] [INFO] [axolotl.core.trainers.base._save:828] [PID:1520] Saving model checkpoint to /workspace/python4-aft-v2-20260813T154138Z-ordered_4ep/experiments/python4/aft_v2/runs/20260813T154138Z/arms/ordered_4ep/train/checkpoints/checkpoint-128 |
|
{'train_runtime': '1318', 'train_samples_per_second': '3.108', 'train_steps_per_second': '0.097', 'train_loss': '0.1855', 'memory/max_active (GiB)': '56.24', 'memory/max_allocated (GiB)': '56.24', 'memory/device_reserved (GiB)': '65.73', 'epoch': '4', 'tokens/train_per_sec_per_gpu': '0'} |
|
100%|██████████| 128/128 [21:57<00:00, 10.69s/it]
100%|██████████| 128/128 [21:57<00:00, 10.30s/it] |
| [2026-08-13 16:11:47,678] [INFO] [axolotl.train.save_trained_model:267] [PID:1520] Training completed! Saving trained model to /workspace/python4-aft-v2-20260813T154138Z-ordered_4ep/experiments/python4/aft_v2/runs/20260813T154138Z/arms/ordered_4ep/train/checkpoints. |
| [2026-08-13 16:11:48,913] [INFO] [axolotl.train.save_trained_model:388] [PID:1520] Model successfully saved to /workspace/python4-aft-v2-20260813T154138Z-ordered_4ep/experiments/python4/aft_v2/runs/20260813T154138Z/arms/ordered_4ep/train/checkpoints |
|
|
