[2026-03-16 19:06:45,455] [DEBUG] [axolotl.utils.config.log_gpu_memory_usage:127] [PID:213] baseline 0.000GB ()
[2026-03-16 19:06:45,456] [INFO] [axolotl.cli.config.load_cfg:340] [PID:213] config:
{
  "activation_offloading": false,
  "axolotl_config_path": "qwen3-sft-stmt-tk.yml",
  "base_model": "Qwen/Qwen3-8B",
  "base_model_config": "Qwen/Qwen3-8B",
  "batch_size": 16,
  "bf16": true,
  "capabilities": {
    "bf16": true,
    "compute_capability": "sm_90",
    "fp8": true,
    "n_gpu": 8,
    "n_node": 1
  },
  "chat_template": "qwen3",
  "chat_template_kwargs": {
    "enable_thinking": false
  },
  "context_parallel_size": 1,
  "dataloader_num_workers": 8,
  "dataloader_pin_memory": true,
  "dataloader_prefetch_factor": 256,
  "dataset_num_proc": 192,
  "datasets": [
    {
      "message_property_mappings": {
        "content": "content",
        "role": "role"
      },
      "path": "xiaolesu/lean4-sft-stmt-tk",
      "split": "train",
      "trust_remote_code": false,
      "type": "alpaca"
    }
  ],
  "ddp": true,
  "device": "cuda:0",
  "device_map": {
    "": 0
  },
  "dion_rank_fraction": 1.0,
  "dion_rank_multiple_of": 1,
  "eaft_alpha": 1.0,
  "eaft_k": 20,
  "env_capabilities": {
    "torch_version": "2.9.1"
  },
  "eval_batch_size": 2,
  "eval_causal_lm_metrics": [
    "sacrebleu",
    "comet",
    "ter",
    "chrf"
  ],
  "eval_max_new_tokens": 128,
  "eval_sample_packing": true,
  "eval_table_size": 0,
  "evals_per_epoch": 10,
  "experimental_skip_move_to_device": true,
  "flex_attention": true,
  "flex_attn_compile_kwargs": {
    "dynamic": false,
    "mode": "max-autotune-no-cudagraphs"
  },
  "fp16": false,
  "fsdp": [
    "full_shard",
    "auto_wrap"
  ],
  "fsdp_config": {
    "activation_checkpointing": true,
    "auto_wrap_policy": "TRANSFORMER_BASED_WRAP",
    "cpu_ram_efficient_loading": true,
    "fsdp_version": 2,
    "offload_params": false,
    "reshard_after_forward": true,
    "state_dict_type": "FULL_STATE_DICT",
    "transformer_layer_cls_to_wrap": "Qwen3DecoderLayer"
  },
  "fsdp_version": 2,
  "generate_samples": false,
  "generation_do_sample": true,
  "generation_max_new_tokens": 50,
  "generation_prompt_ratio": 0.5,
  "generation_temperature": 0.7,
  "gradient_accumulation_steps": 1,
  "gradient_checkpointing": false,
  "include_tkps": true,
  "learning_rate": 1e-05,
  "liger_fused_linear_cross_entropy": true,
  "liger_glu_activation": true,
  "liger_layer_norm": true,
  "liger_rms_norm": true,
  "liger_rope": true,
  "lisa_layers_attribute": "model.layers",
  "load_best_model_at_end": false,
  "load_in_4bit": false,
  "load_in_8bit": false,
  "local_rank": 0,
  "logging_steps": 5,
  "lora_dropout": 0.0,
  "loraplus_lr_embedding": 1e-06,
  "lr_scheduler": "cosine",
  "mean_resizing_embeddings": false,
  "micro_batch_size": 2,
  "model_config_type": "qwen3",
  "num_epochs": 2.0,
  "num_generation_samples": 3,
  "optimizer": "adamw_torch_fused",
  "otel_metrics_host": "localhost",
  "otel_metrics_port": 8000,
  "output_dir": "./outputs/qwen3-sft-stmt-tk/",
  "pad_to_sequence_len": true,
  "plugins": [
    "axolotl.integrations.liger.LigerPlugin"
  ],
  "pretrain_multipack_attn": true,
  "profiler_steps_start": 0,
  "qlora_sharded_model_loading": false,
  "quantize_moe_experts": false,
  "ray_num_workers": 1,
  "resources_per_worker": {
    "GPU": 1
  },
  "sample_packing": true,
  "sample_packing_bin_size": 200,
  "sample_packing_group_size": 100000,
  "save_only_model": false,
  "save_safetensors": true,
  "save_steps": 0.05,
  "save_total_limit": 3,
  "saves_per_epoch": 10,
  "sequence_len": 8192,
  "shuffle_before_merging_datasets": false,
  "shuffle_merged_datasets": true,
  "skip_prepare_dataset": false,
  "streaming_multipack_buffer_size": 10000,
  "strict": false,
  "tensor_parallel_size": 1,
  "tf32": true,
  "tiled_mlp_use_original_mlp": true,
  "tokenizer_config": "Qwen/Qwen3-8B",
  "tokenizer_save_jinja_files": true,
  "torch_dtype": "torch.bfloat16",
  "train_on_inputs": false,
  "trl": {
    "log_completions": false,
    "mask_truncated_completions": false,
    "ref_model_mixup_alpha": 0.9,
    "ref_model_sync_steps": 64,
    "scale_rewards": true,
    "sync_ref_model": false,
    "use_vllm": false,
    "vllm_server_host": "0.0.0.0",
    "vllm_server_port": 8000
  },
  "use_otel_metrics": false,
  "use_ray": false,
  "use_wandb": true,
  "val_set_size": 0.0,
  "vllm": {
    "device": "auto",
    "dtype": "auto",
    "gpu_memory_utilization": 0.9,
    "host": "0.0.0.0",
    "port": 8000
  },
  "wandb_name": "qwen3-8b-tk-run1",
  "wandb_project": "qwen3-sft-stmt-tk",
  "warmup_ratio": 0.1,
  "weight_decay": 0.0,
  "world_size": 8
}
[2026-03-16 19:06:47,178] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:299] [PID:213] EOS: 151645 / <|im_end|>
[2026-03-16 19:06:47,178] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:300] [PID:213] BOS: None / None
[2026-03-16 19:06:47,178] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:301] [PID:213] PAD: 151643 / <|endoftext|>
[2026-03-16 19:06:47,178] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:302] [PID:213] UNK: None / None
[2026-03-16 19:08:33,239] [INFO] [axolotl.utils.data.shared.load_preprocessed_dataset:480] [PID:213] Unable to find prepared dataset in last_run_prepared/a7f1540a69de94eaad2000d92fac4b11
[2026-03-16 19:08:33,239] [INFO] [axolotl.utils.data.sft._load_raw_datasets:320] [PID:213] Loading raw datasets...
[2026-03-16 19:08:33,239] [WARNING] [axolotl.utils.data.sft._load_raw_datasets:322] [PID:213] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`.

Fetching 0 files: 0it [00:00, ?it/s]
Fetching 0 files: 0it [00:00, ?it/s]
[2026-03-16 19:08:34,675] [INFO] [axolotl.utils.data.wrappers.get_dataset_wrapper:87] [PID:213] Loading dataset: xiaolesu/lean4-sft-stmt-tk with base_type: alpaca and prompt_style: None
[2026-03-16 19:08:36,088] [INFO] [axolotl.utils.data.utils._log_dataset_stats:212] [PID:213] min_input_len: 205
[2026-03-16 19:08:36,088] [INFO] [axolotl.utils.data.utils._log_dataset_stats:213] [PID:213] max_input_len: 9159

Dropping Invalid Sequences (<None or >8192) (num_proc=192):   0%|          | 0/11192 [00:00<?, ? examples/s]
Dropping Invalid Sequences (<None or >8192) (num_proc=192):   1%|          | 59/11192 [00:02<06:34, 28.25 examples/s]
Dropping Invalid Sequences (<None or >8192) (num_proc=192):   3%|▎         | 295/11192 [00:02<01:02, 175.65 examples/s]
Dropping Invalid Sequences (<None or >8192) (num_proc=192):   6%|▌         | 649/11192 [00:02<00:23, 453.06 examples/s]
Dropping Invalid Sequences (<None or >8192) (num_proc=192):   8%|▊         | 885/11192 [00:02<00:16, 634.46 examples/s]
Dropping Invalid Sequences (<None or >8192) (num_proc=192):  10%|█         | 1121/11192 [00:02<00:11, 849.04 examples/s]
Dropping Invalid Sequences (<None or >8192) (num_proc=192):  13%|█▎        | 1416/11192 [00:02<00:08, 1166.00 examples/s]
Dropping Invalid Sequences (<None or >8192) (num_proc=192):  15%|█▌        | 1711/11192 [00:02<00:06, 1480.17 examples/s]
Dropping Invalid Sequences (<None or >8192) (num_proc=192):  18%|█▊        | 2006/11192 [00:02<00:05, 1697.58 examples/s]
Dropping Invalid Sequences (<None or >8192) (num_proc=192):  21%|██        | 2301/11192 [00:02<00:04, 1949.74 examples/s]
Dropping Invalid Sequences (<None or >8192) (num_proc=192):  23%|██▎       | 2596/11192 [00:03<00:04, 2145.10 examples/s]
Dropping Invalid Sequences (<None or >8192) (num_proc=192):  26%|██▌       | 2891/11192 [00:03<00:03, 2324.57 examples/s]
Dropping Invalid Sequences (<None or >8192) (num_proc=192):  29%|██▉       | 3245/11192 [00:03<00:03, 2566.75 examples/s]
Dropping Invalid Sequences (<None or >8192) (num_proc=192):  70%|██████▉   | 7828/11192 [00:03<00:00, 14035.00 examples/s]
Dropping Invalid Sequences (<None or >8192) (num_proc=192): 100%|██████████| 11192/11192 [00:04<00:00, 2753.84 examples/s]
[2026-03-16 19:08:41,123] [INFO] [axolotl.utils.data.utils._drop_outside_range:306] [PID:213] Dropped 362 sequences outside valid range ([None, 8192])

Drop Samples with Zero Trainable Tokens (num_proc=192):   0%|          | 0/10830 [00:00<?, ? examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=192):   1%|          | 57/10830 [00:02<06:27, 27.78 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=192):   3%|▎         | 285/10830 [00:02<01:00, 173.64 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=192):   4%|▍         | 456/10830 [00:02<00:34, 299.77 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=192):   6%|▋         | 684/10830 [00:02<00:20, 506.62 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=192):   8%|▊         | 912/10830 [00:02<00:13, 736.95 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=192):  11%|█         | 1140/10830 [00:02<00:10, 947.17 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=192):  13%|█▎        | 1368/10830 [00:02<00:08, 1094.03 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=192):  15%|█▍        | 1596/10830 [00:02<00:07, 1269.49 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=192):  17%|█▋        | 1824/10830 [00:02<00:06, 1437.65 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=192):  19%|█▉        | 2052/10830 [00:03<00:05, 1614.63 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=192):  21%|██        | 2280/10830 [00:03<00:05, 1635.72 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=192):  23%|██▎       | 2508/10830 [00:03<00:04, 1732.21 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=192):  25%|██▌       | 2736/10830 [00:03<00:04, 1721.60 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=192):  27%|██▋       | 2964/10830 [00:03<00:04, 1703.27 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=192):  29%|██▉       | 3192/10830 [00:03<00:04, 1798.77 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=192):  32%|███▏      | 3477/10830 [00:03<00:03, 1958.86 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=192):  34%|███▍      | 3705/10830 [00:03<00:03, 2037.08 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=192):  36%|███▋      | 3933/10830 [00:04<00:03, 2067.96 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=192):  38%|███▊      | 4161/10830 [00:04<00:03, 2091.19 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=192):  41%|████      | 4389/10830 [00:04<00:05, 1127.36 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=192):  43%|████▎     | 4670/10830 [00:04<00:04, 1385.39 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=192):  45%|████▌     | 4894/10830 [00:04<00:04, 1432.10 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=192):  69%|██████▉   | 7526/10830 [00:04<00:00, 6499.14 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=192): 100%|██████████| 10830/10830 [00:05<00:00, 1931.57 examples/s]

Add position_id column (Sample Packing) (num_proc=192):   0%|          | 0/10830 [00:00<?, ? examples/s]
Add position_id column (Sample Packing) (num_proc=192):   1%|          | 57/10830 [00:02<06:33, 27.40 examples/s]
Add position_id column (Sample Packing) (num_proc=192):   2%|▏         | 228/10830 [00:02<01:18, 135.14 examples/s]
Add position_id column (Sample Packing) (num_proc=192):   4%|▍         | 456/10830 [00:02<00:33, 310.31 examples/s]
Add position_id column (Sample Packing) (num_proc=192):   8%|▊         | 912/10830 [00:02<00:14, 692.10 examples/s]
Add position_id column (Sample Packing) (num_proc=192):  11%|█         | 1140/10830 [00:02<00:11, 858.26 examples/s]
Add position_id column (Sample Packing) (num_proc=192):  13%|█▎        | 1368/10830 [00:02<00:09, 1027.56 examples/s]
Add position_id column (Sample Packing) (num_proc=192):  15%|█▍        | 1596/10830 [00:02<00:07, 1182.55 examples/s]
Add position_id column (Sample Packing) (num_proc=192):  17%|█▋        | 1881/10830 [00:02<00:06, 1425.26 examples/s]
Add position_id column (Sample Packing) (num_proc=192):  20%|██        | 2166/10830 [00:03<00:05, 1604.97 examples/s]
Add position_id column (Sample Packing) (num_proc=192):  22%|██▏       | 2394/10830 [00:03<00:04, 1738.29 examples/s]
Add position_id column (Sample Packing) (num_proc=192):  25%|██▍       | 2679/10830 [00:03<00:04, 1951.23 examples/s]
Add position_id column (Sample Packing) (num_proc=192):  63%|██████▎   | 6854/10830 [00:03<00:00, 11681.66 examples/s]
Add position_id column (Sample Packing) (num_proc=192): 100%|██████████| 10830/10830 [00:04<00:00, 2621.72 examples/s]

Saving the dataset (0/42 shards):   0%|          | 0/10830 [00:00<?, ? examples/s]
Saving the dataset (0/42 shards):   2%|▏         | 258/10830 [00:00<00:22, 464.02 examples/s]
Saving the dataset (1/42 shards):   2%|▏         | 258/10830 [00:00<00:22, 464.02 examples/s]
Saving the dataset (2/42 shards):   7%|▋         | 774/10830 [00:00<00:21, 464.02 examples/s]
Saving the dataset (3/42 shards):   7%|▋         | 774/10830 [00:00<00:21, 464.02 examples/s]
Saving the dataset (4/42 shards):  14%|█▍        | 1548/10830 [00:00<00:20, 464.02 examples/s]
Saving the dataset (5/42 shards):  14%|█▍        | 1548/10830 [00:00<00:20, 464.02 examples/s]
Saving the dataset (6/42 shards):  17%|█▋        | 1806/10830 [00:00<00:19, 464.02 examples/s]
Saving the dataset (7/42 shards):  19%|█▉        | 2064/10830 [00:00<00:18, 464.02 examples/s]
Saving the dataset (8/42 shards):  21%|██▏       | 2322/10830 [00:00<00:18, 464.02 examples/s]
Saving the dataset (9/42 shards):  21%|██▏       | 2322/10830 [00:00<00:18, 464.02 examples/s]
Saving the dataset (10/42 shards):  26%|██▌       | 2838/10830 [00:00<00:17, 464.02 examples/s]
Saving the dataset (11/42 shards):  29%|██▊       | 3096/10830 [00:00<00:16, 464.02 examples/s]
Saving the dataset (12/42 shards):  31%|███       | 3354/10830 [00:00<00:16, 464.02 examples/s]
Saving the dataset (13/42 shards):  33%|███▎      | 3612/10830 [00:00<00:15, 464.02 examples/s]
Saving the dataset (14/42 shards):  33%|███▎      | 3612/10830 [00:00<00:15, 464.02 examples/s]
Saving the dataset (15/42 shards):  38%|███▊      | 4128/10830 [00:00<00:14, 464.02 examples/s]
Saving the dataset (16/42 shards):  40%|████      | 4386/10830 [00:00<00:13, 464.02 examples/s]
Saving the dataset (17/42 shards):  40%|████      | 4386/10830 [00:00<00:13, 464.02 examples/s]
Saving the dataset (18/42 shards):  45%|████▌     | 4902/10830 [00:00<00:12, 464.02 examples/s]
Saving the dataset (19/42 shards):  48%|████▊     | 5160/10830 [00:00<00:12, 464.02 examples/s]
Saving the dataset (20/42 shards):  48%|████▊     | 5160/10830 [00:00<00:12, 464.02 examples/s]
Saving the dataset (21/42 shards):  52%|█████▏    | 5676/10830 [00:00<00:11, 464.02 examples/s]
Saving the dataset (22/42 shards):  52%|█████▏    | 5676/10830 [00:00<00:11, 464.02 examples/s]
Saving the dataset (23/42 shards):  55%|█████▍    | 5934/10830 [00:00<00:10, 464.02 examples/s]
Saving the dataset (24/42 shards):  57%|█████▋    | 6192/10830 [00:00<00:09, 464.02 examples/s]
Saving the dataset (25/42 shards):  64%|██████▍   | 6966/10830 [00:00<00:08, 464.02 examples/s]
Saving the dataset (26/42 shards):  64%|██████▍   | 6966/10830 [00:00<00:08, 464.02 examples/s]
Saving the dataset (27/42 shards):  64%|██████▍   | 6966/10830 [00:00<00:08, 464.02 examples/s]
Saving the dataset (28/42 shards):  67%|██████▋   | 7224/10830 [00:00<00:07, 464.02 examples/s]
Saving the dataset (29/42 shards):  74%|███████▍  | 7998/10830 [00:00<00:06, 464.02 examples/s]
Saving the dataset (30/42 shards):  74%|███████▍  | 7998/10830 [00:00<00:06, 464.02 examples/s]
Saving the dataset (31/42 shards):  74%|███████▍  | 7998/10830 [00:00<00:06, 464.02 examples/s]
Saving the dataset (32/42 shards):  79%|███████▊  | 8514/10830 [00:00<00:04, 464.02 examples/s]
Saving the dataset (33/42 shards):  81%|████████  | 8772/10830 [00:00<00:04, 464.02 examples/s]
Saving the dataset (34/42 shards):  81%|████████  | 8772/10830 [00:00<00:04, 464.02 examples/s]
Saving the dataset (35/42 shards):  83%|████████▎ | 9030/10830 [00:00<00:03, 464.02 examples/s]
Saving the dataset (36/42 shards):  88%|████████▊ | 9545/10830 [00:00<00:02, 464.02 examples/s]
Saving the dataset (37/42 shards):  88%|████████▊ | 9545/10830 [00:00<00:02, 464.02 examples/s]
Saving the dataset (38/42 shards):  91%|█████████ | 9802/10830 [00:00<00:02, 464.02 examples/s]
Saving the dataset (39/42 shards):  95%|█████████▌| 10316/10830 [00:00<00:01, 464.02 examples/s]
Saving the dataset (40/42 shards):  95%|█████████▌| 10316/10830 [00:00<00:01, 464.02 examples/s]
Saving the dataset (41/42 shards):  98%|█████████▊| 10573/10830 [00:00<00:00, 464.02 examples/s]
Saving the dataset (42/42 shards): 100%|██████████| 10830/10830 [00:00<00:00, 464.02 examples/s]
Saving the dataset (42/42 shards): 100%|██████████| 10830/10830 [00:00<00:00, 16314.56 examples/s]
[2026-03-16 19:08:54,045] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:420] [PID:213] total_num_tokens: 33_957_071
[2026-03-16 19:08:54,340] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:438] [PID:213] `total_supervised_tokens: 32_028_150`
[2026-03-16 19:08:55,893] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:213] generate_batches time: 0.7050187587738037
[2026-03-16 19:11:05,467] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:213] gather_len_batches: [2148, 2146, 2148, 2145, 2146, 2146, 2148, 2145]
[2026-03-16 19:11:06,172] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:495] [PID:213] data_loader_len: 268
[2026-03-16 19:11:06,189] [INFO] [axolotl.utils.trainer.calc_sample_packing_eff_est:504] [PID:213] sample_packing_eff_est across ranks: [0.9646614789962769, 0.9657852649688721, 0.9646614789962769, 0.9657852649688721, 0.9648860096931458, 0.9648860096931458, 0.9653354287147522, 0.9657852649688721]
[2026-03-16 19:11:06,190] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:516] [PID:213] sample_packing_eff_est: 0.97
[2026-03-16 19:11:06,190] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:521] [PID:213] total_num_steps: 536
[2026-03-16 19:11:06,192] [INFO] [axolotl.utils.data.sft._prepare_standard_dataset:121] [PID:213] Maximum number of steps set at 536
[2026-03-16 19:11:06,242] [DEBUG] [axolotl.train.setup_model_and_tokenizer:70] [PID:213] loading tokenizer... Qwen/Qwen3-8B
[2026-03-16 19:11:07,694] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:299] [PID:213] EOS: 151645 / <|im_end|>
[2026-03-16 19:11:07,694] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:300] [PID:213] BOS: None / None
[2026-03-16 19:11:07,694] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:301] [PID:213] PAD: 151643 / <|endoftext|>
[2026-03-16 19:11:07,694] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:302] [PID:213] UNK: None / None
[2026-03-16 19:11:07,694] [DEBUG] [axolotl.train.setup_model_and_tokenizer:81] [PID:213] Loading model
[2026-03-16 19:11:07,808] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_evaluation_loop:91] [PID:213] Patched Trainer.evaluation_loop with nanmean loss calculation
[2026-03-16 19:11:07,809] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_maybe_log_save_evaluate:142] [PID:213] Patched Trainer._maybe_log_save_evaluate with nanmean loss calculation
[2026-03-16 19:11:07,811] [INFO] [axolotl.loaders.patch_manager._apply_multipack_patches:400] [PID:213] Applying multipack dataloader patch for sample packing...
[2026-03-16 19:11:09,375] [INFO] [axolotl.integrations.liger.plugin.pre_model_load:104] [PID:213] Applying LIGER to qwen3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'swiglu': True}

Loading weights:   0%|          | 0/399 [00:00<?, ?it/s]
Loading weights: 100%|██████████| 399/399 [00:00<00:00, 9671.84it/s]
[2026-03-16 19:11:09,882] [INFO] [axolotl.loaders.model._configure_embedding_dtypes:359] [PID:213] Converting modules to torch.bfloat16
[2026-03-16 19:11:09,885] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:213] Memory usage after model load 0.000GB (+0.000GB allocated, +0.002GB reserved)
[2026-03-16 19:11:11,696] [WARNING] [accelerate.utils.dataclasses.__post_init__:1992] [PID:213] sharding_strategy is deprecated in favor of reshard_after_forward. This will be removed in a future version of Accelerate.Multiple deprecation warnings due to FSDP2 conversion:
sync_module_states is obsolete in FSDP2, as it is not needed anymore.Setting sync_module_states to None.
[2026-03-16 19:11:12,192] [INFO] [axolotl.train.save_initial_configs:417] [PID:213] Pre-saving tokenizer to ./outputs/qwen3-sft-stmt-tk/...
[2026-03-16 19:11:12,283] [INFO] [axolotl.train.save_initial_configs:422] [PID:213] Pre-saving model config to ./outputs/qwen3-sft-stmt-tk/...
[2026-03-16 19:11:12,286] [INFO] [axolotl.train.execute_training:218] [PID:213] Starting trainer...
[2026-03-16 19:11:14,793] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:213] generate_batches time: 0.9547648429870605
[2026-03-16 19:11:14,796] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:213] gather_len_batches: [2103, 2104, 2104, 2104, 2103, 2104, 2106, 2104]
[2026-03-16 19:11:15,013] [INFO] [axolotl.monkeypatch.accelerate.fsdp2.fsdp2_load_full_state_dict:34] [PID:213] Broadcasting full state dict to all ranks...
[2026-03-16 19:11:22,269] [DEBUG] [axolotl.monkeypatch.accelerate.fsdp2.fsdp2_load_full_state_dict:86] [PID:213] Time taken to load full state dict: 7.26 seconds
[2026-03-16 19:11:22,270] [DEBUG] [axolotl.monkeypatch.accelerate.fsdp2.log_gpu_memory_usage:127] [PID:213] Memory usage after broadcasting full state dict 3.067GB (+3.067GB allocated, +3.178GB reserved)
wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
wandb: Currently logged in as: suxiaole0223 (suxiaole) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
wandb: setting up run kje10pck
wandb: Tracking run with wandb version 0.25.1
wandb: Run data is saved locally in /workspace/axolotl-workspace/wandb/run-20260316_191122-kje10pck
wandb: Run `wandb offline` to turn off syncing.
wandb: Syncing run qwen3-8b-tk-run1
wandb: ⭐️ View project at https://wandb.ai/suxiaole/qwen3-sft-stmt-tk
wandb: 🚀 View run at https://wandb.ai/suxiaole/qwen3-sft-stmt-tk/runs/kje10pck
wandb: WARNING Saving files without folders. If you want to preserve subdirectories pass base_path to wandb.save, i.e. wandb.save("/mnt/folder/file.h5", base_path="/mnt")
wandb: WARNING Symlinked 1 file into the W&B run directory; call wandb.save again to sync new files.
[2026-03-16 19:11:25,554] [INFO] [axolotl.utils.callbacks.on_train_begin:757] [PID:213] The Axolotl config has been saved to the WandB run under files.

  0%|          | 0/536 [00:00<?, ?it/s][2026-03-16 19:11:57,210] [WARNING] [py.warnings._showwarnmsg:110] [PID:213] /root/miniconda3/envs/py3.11/lib/python3.11/site-packages/torch/nn/attention/flex_attention.py:1622: FutureWarning: return_lse is deprecated and will be removed in v2.10. Please use return_aux=AuxRequest(lse=True) instead.
  _warn_once(


  0%|          | 1/536 [00:40<6:03:21, 40.75s/it]
  0%|          | 2/536 [00:43<2:42:00, 18.20s/it]
  1%|          | 3/536 [00:45<1:37:15, 10.95s/it]
  1%|          | 4/536 [00:47<1:07:23,  7.60s/it]
  1%|          | 5/536 [00:50<50:28,  5.70s/it]  
                                               
{'loss': '0.8667', 'grad_norm': '2.609', 'learning_rate': '7.547e-07', 'ppl': '2.379', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6531', 'tokens/total': 655360, 'tokens/trainable': 611049, 'epoch': '0.01908'}

  1%|          | 5/536 [00:50<50:28,  5.70s/it]
  1%|          | 6/536 [00:52<40:15,  4.56s/it]
  1%|▏         | 7/536 [00:55<34:02,  3.86s/it]
  1%|▏         | 8/536 [00:57<30:00,  3.41s/it]
  2%|▏         | 9/536 [00:59<26:45,  3.05s/it]
  2%|▏         | 10/536 [01:02<24:45,  2.82s/it]
                                                
{'loss': '0.8307', 'grad_norm': '2.5', 'learning_rate': '1.698e-06', 'ppl': '2.295', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6647', 'tokens/total': 1310720, 'tokens/trainable': 1224548, 'epoch': '0.03817'}

  2%|▏         | 10/536 [01:02<24:45,  2.82s/it]
  2%|▏         | 11/536 [01:04<23:13,  2.65s/it]
  2%|▏         | 12/536 [01:06<22:04,  2.53s/it]
  2%|▏         | 13/536 [01:08<21:32,  2.47s/it]
  3%|▎         | 14/536 [01:11<21:27,  2.47s/it]
  3%|▎         | 15/536 [01:13<21:28,  2.47s/it]
                                                
{'loss': '0.8487', 'grad_norm': '2.453', 'learning_rate': '2.642e-06', 'ppl': '2.337', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6160', 'tokens/total': 1966080, 'tokens/trainable': 1834432, 'epoch': '0.05725'}

  3%|▎         | 15/536 [01:13<21:28,  2.47s/it]
  3%|▎         | 16/536 [01:16<21:18,  2.46s/it]
  3%|▎         | 17/536 [01:18<20:51,  2.41s/it]
  3%|▎         | 18/536 [01:20<20:44,  2.40s/it]
  4%|▎         | 19/536 [01:23<21:59,  2.55s/it]
  4%|▎         | 20/536 [01:26<21:40,  2.52s/it]
                                                
{'loss': '0.7713', 'grad_norm': '1.898', 'learning_rate': '3.585e-06', 'ppl': '2.163', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6256', 'tokens/total': 2621440, 'tokens/trainable': 2448388, 'epoch': '0.07634'}

  4%|▎         | 20/536 [01:26<21:40,  2.52s/it]
  4%|▍         | 21/536 [01:28<21:23,  2.49s/it]
  4%|▍         | 22/536 [01:31<20:49,  2.43s/it]
  4%|▍         | 23/536 [01:33<20:37,  2.41s/it]
  4%|▍         | 24/536 [01:35<20:37,  2.42s/it]
  5%|▍         | 25/536 [01:38<20:01,  2.35s/it]
                                                
{'loss': '0.7452', 'grad_norm': '1.273', 'learning_rate': '4.528e-06', 'ppl': '2.107', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6954', 'tokens/total': 3276800, 'tokens/trainable': 3060985, 'epoch': '0.09542'}

  5%|▍         | 25/536 [01:38<20:01,  2.35s/it]
  5%|▍         | 26/536 [01:40<19:41,  2.32s/it]
  5%|▌         | 27/536 [01:42<19:26,  2.29s/it][2026-03-16 19:13:17,483] [INFO] [axolotl.core.trainers.base._save:721] [PID:213] Saving model checkpoint to ./outputs/qwen3-sft-stmt-tk/checkpoint-27


Writing model shards:   0%|          | 0/1 [00:00<?, ?it/s]

Writing model shards: 100%|██████████| 1/1 [00:16<00:00, 16.48s/it]
Writing model shards: 100%|██████████| 1/1 [00:16<00:00, 16.48s/it]

  5%|▌         | 28/536 [03:16<4:11:47, 29.74s/it]
  5%|▌         | 29/536 [03:18<3:01:39, 21.50s/it]
  6%|▌         | 30/536 [03:20<2:12:38, 15.73s/it]
                                                  
{'loss': '0.718', 'grad_norm': '0.7695', 'learning_rate': '5.472e-06', 'ppl': '2.05', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6706', 'tokens/total': 3932160, 'tokens/trainable': 3670695, 'epoch': '0.1145'}

  6%|▌         | 30/536 [03:20<2:12:38, 15.73s/it]
  6%|▌         | 31/536 [03:23<1:38:27, 11.70s/it]
  6%|▌         | 32/536 [03:25<1:14:38,  8.89s/it]
  6%|▌         | 33/536 [03:27<57:48,  6.90s/it]  
  6%|▋         | 34/536 [03:29<46:07,  5.51s/it]
  7%|▋         | 35/536 [03:32<37:56,  4.54s/it]
                                                
{'loss': '0.6699', 'grad_norm': '0.6406', 'learning_rate': '6.415e-06', 'ppl': '1.954', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6770', 'tokens/total': 4587520, 'tokens/trainable': 4284736, 'epoch': '0.1336'}

  7%|▋         | 35/536 [03:32<37:56,  4.54s/it]
  7%|▋         | 36/536 [03:34<32:19,  3.88s/it]
  7%|▋         | 37/536 [03:37<28:45,  3.46s/it]
  7%|▋         | 38/536 [03:39<26:05,  3.14s/it]
  7%|▋         | 39/536 [03:41<24:10,  2.92s/it]
  7%|▋         | 40/536 [03:44<22:31,  2.72s/it]
                                                
{'loss': '0.6393', 'grad_norm': '0.418', 'learning_rate': '7.358e-06', 'ppl': '1.895', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6668', 'tokens/total': 5242880, 'tokens/trainable': 4896504, 'epoch': '0.1527'}

  7%|▋         | 40/536 [03:44<22:31,  2.72s/it]
  8%|▊         | 41/536 [03:46<21:24,  2.59s/it]
  8%|▊         | 42/536 [03:48<20:36,  2.50s/it]
  8%|▊         | 43/536 [03:51<20:06,  2.45s/it]
  8%|▊         | 44/536 [03:53<19:38,  2.39s/it]
  8%|▊         | 45/536 [03:55<19:17,  2.36s/it]
                                                
{'loss': '0.5953', 'grad_norm': '0.3594', 'learning_rate': '8.302e-06', 'ppl': '1.814', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6726', 'tokens/total': 5898240, 'tokens/trainable': 5505933, 'epoch': '0.1718'}

  8%|▊         | 45/536 [03:55<19:17,  2.36s/it]
  9%|▊         | 46/536 [03:57<19:17,  2.36s/it]
  9%|▉         | 47/536 [04:00<19:01,  2.33s/it]
  9%|▉         | 48/536 [04:02<19:02,  2.34s/it]
  9%|▉         | 49/536 [04:04<19:02,  2.35s/it]
  9%|▉         | 50/536 [04:07<18:55,  2.34s/it]
                                                
{'loss': '0.5779', 'grad_norm': '0.332', 'learning_rate': '9.245e-06', 'ppl': '1.782', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6574', 'tokens/total': 6553600, 'tokens/trainable': 6116643, 'epoch': '0.1908'}

  9%|▉         | 50/536 [04:07<18:55,  2.34s/it]
 10%|▉         | 51/536 [04:09<18:46,  2.32s/it]
 10%|▉         | 52/536 [04:11<18:33,  2.30s/it]
 10%|▉         | 53/536 [04:14<18:19,  2.28s/it]
 10%|█         | 54/536 [04:16<18:23,  2.29s/it][2026-03-16 19:15:50,860] [INFO] [axolotl.core.trainers.base._save:721] [PID:213] Saving model checkpoint to ./outputs/qwen3-sft-stmt-tk/checkpoint-54


Writing model shards:   0%|          | 0/1 [00:00<?, ?it/s]

Writing model shards: 100%|██████████| 1/1 [00:16<00:00, 16.65s/it]
Writing model shards: 100%|██████████| 1/1 [00:16<00:00, 16.65s/it]

 10%|█         | 55/536 [05:48<3:55:25, 29.37s/it]
                                                  
{'loss': '0.5579', 'grad_norm': '0.2793', 'learning_rate': '1e-05', 'ppl': '1.747', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '4847', 'tokens/total': 7208960, 'tokens/trainable': 6728061, 'epoch': '0.2099'}

 10%|█         | 55/536 [05:48<3:55:25, 29.37s/it]
 10%|█         | 56/536 [05:51<2:50:56, 21.37s/it]
 11%|█         | 57/536 [05:54<2:05:27, 15.72s/it]
 11%|█         | 58/536 [05:56<1:33:09, 11.69s/it]
 11%|█         | 59/536 [05:58<1:11:13,  8.96s/it]
 11%|█         | 60/536 [06:01<55:01,  6.94s/it]  
                                                
{'loss': '0.5485', 'grad_norm': '0.2773', 'learning_rate': '9.996e-06', 'ppl': '1.731', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6813', 'tokens/total': 7864320, 'tokens/trainable': 7336524, 'epoch': '0.229'}

 11%|█         | 60/536 [06:01<55:01,  6.94s/it]
 11%|█▏        | 61/536 [06:03<43:45,  5.53s/it]
 12%|█▏        | 62/536 [06:05<36:05,  4.57s/it]
 12%|█▏        | 63/536 [06:08<30:31,  3.87s/it]
 12%|█▏        | 64/536 [06:10<26:38,  3.39s/it]
 12%|█▏        | 65/536 [06:12<24:01,  3.06s/it]
                                                
{'loss': '0.5385', 'grad_norm': '0.2734', 'learning_rate': '9.987e-06', 'ppl': '1.713', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6565', 'tokens/total': 8519680, 'tokens/trainable': 7944984, 'epoch': '0.2481'}

 12%|█▏        | 65/536 [06:12<24:01,  3.06s/it]
 12%|█▏        | 66/536 [06:14<22:09,  2.83s/it]
 12%|█▎        | 67/536 [06:17<21:06,  2.70s/it]
 13%|█▎        | 68/536 [06:19<20:07,  2.58s/it]
 13%|█▎        | 69/536 [06:21<19:21,  2.49s/it]
 13%|█▎        | 70/536 [06:24<19:00,  2.45s/it]
                                                
{'loss': '0.5197', 'grad_norm': '0.2578', 'learning_rate': '9.973e-06', 'ppl': '1.682', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6471', 'tokens/total': 9175040, 'tokens/trainable': 8556200, 'epoch': '0.2672'}

 13%|█▎        | 70/536 [06:24<19:00,  2.45s/it]
 13%|█▎        | 71/536 [06:26<18:33,  2.39s/it]
 13%|█▎        | 72/536 [06:28<18:14,  2.36s/it]
 14%|█▎        | 73/536 [06:31<18:01,  2.34s/it]
 14%|█▍        | 74/536 [06:33<18:02,  2.34s/it]
 14%|█▍        | 75/536 [06:35<17:54,  2.33s/it]
                                                
{'loss': '0.5316', 'grad_norm': '0.3008', 'learning_rate': '9.953e-06', 'ppl': '1.702', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6728', 'tokens/total': 9830400, 'tokens/trainable': 9167282, 'epoch': '0.2863'}

 14%|█▍        | 75/536 [06:35<17:54,  2.33s/it]
 14%|█▍        | 76/536 [06:38<17:54,  2.34s/it]
 14%|█▍        | 77/536 [06:40<18:05,  2.37s/it]
 15%|█▍        | 78/536 [06:43<18:28,  2.42s/it]
 15%|█▍        | 79/536 [06:45<18:05,  2.37s/it]
 15%|█▍        | 80/536 [06:47<17:47,  2.34s/it]
                                                
{'loss': '0.5154', 'grad_norm': '0.3164', 'learning_rate': '9.929e-06', 'ppl': '1.674', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6730', 'tokens/total': 10485760, 'tokens/trainable': 9774908, 'epoch': '0.3053'}

 15%|█▍        | 80/536 [06:47<17:47,  2.34s/it]
 15%|█▌        | 81/536 [06:49<17:39,  2.33s/it][2026-03-16 19:18:24,375] [INFO] [axolotl.core.trainers.base._save:721] [PID:213] Saving model checkpoint to ./outputs/qwen3-sft-stmt-tk/checkpoint-81


Writing model shards:   0%|          | 0/1 [00:00<?, ?it/s]

Writing model shards: 100%|██████████| 1/1 [00:17<00:00, 17.37s/it]
Writing model shards: 100%|██████████| 1/1 [00:17<00:00, 17.37s/it]

 15%|█▌        | 82/536 [08:22<3:43:29, 29.54s/it]
 15%|█▌        | 83/536 [08:25<2:41:12, 21.35s/it]
 16%|█▌        | 84/536 [08:27<1:57:43, 15.63s/it]
 16%|█▌        | 85/536 [08:29<1:27:29, 11.64s/it]
                                                  
{'loss': '0.5143', 'grad_norm': '0.2363', 'learning_rate': '9.899e-06', 'ppl': '1.672', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6604', 'tokens/total': 11141120, 'tokens/trainable': 10388109, 'epoch': '0.3244'}

 16%|█▌        | 85/536 [08:29<1:27:29, 11.64s/it]
 16%|█▌        | 86/536 [08:32<1:06:16,  8.84s/it]
 16%|█▌        | 87/536 [08:34<51:19,  6.86s/it]  
 16%|█▋        | 88/536 [08:36<40:55,  5.48s/it]
 17%|█▋        | 89/536 [08:38<33:37,  4.51s/it]
 17%|█▋        | 90/536 [08:41<28:43,  3.86s/it]
                                                
{'loss': '0.4957', 'grad_norm': '0.2412', 'learning_rate': '9.864e-06', 'ppl': '1.642', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6557', 'tokens/total': 11796480, 'tokens/trainable': 10999678, 'epoch': '0.3435'}

 17%|█▋        | 90/536 [08:41<28:43,  3.86s/it]
 17%|█▋        | 91/536 [08:43<25:17,  3.41s/it]
 17%|█▋        | 92/536 [08:45<22:49,  3.08s/it]
 17%|█▋        | 93/536 [08:48<21:12,  2.87s/it]
 18%|█▊        | 94/536 [08:50<19:44,  2.68s/it]
 18%|█▊        | 95/536 [08:52<19:22,  2.64s/it]
                                                
{'loss': '0.509', 'grad_norm': '0.2236', 'learning_rate': '9.823e-06', 'ppl': '1.664', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '5974', 'tokens/total': 12451840, 'tokens/trainable': 11609345, 'epoch': '0.3626'}

 18%|█▊        | 95/536 [08:52<19:22,  2.64s/it]
 18%|█▊        | 96/536 [08:55<18:35,  2.54s/it]
 18%|█▊        | 97/536 [08:57<18:01,  2.46s/it]
 18%|█▊        | 98/536 [09:00<19:09,  2.62s/it]
 18%|█▊        | 99/536 [09:03<19:00,  2.61s/it]
 19%|█▊        | 100/536 [09:05<18:16,  2.51s/it]
                                                 
{'loss': '0.4925', 'grad_norm': '0.2451', 'learning_rate': '9.778e-06', 'ppl': '1.636', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6696', 'tokens/total': 13107200, 'tokens/trainable': 12218448, 'epoch': '0.3817'}

 19%|█▊        | 100/536 [09:05<18:16,  2.51s/it]
 19%|█▉        | 101/536 [09:07<17:50,  2.46s/it]
 19%|█▉        | 102/536 [09:09<17:19,  2.40s/it]
 19%|█▉        | 103/536 [09:12<16:59,  2.35s/it]
 19%|█▉        | 104/536 [09:14<16:47,  2.33s/it]
 20%|█▉        | 105/536 [09:16<16:34,  2.31s/it]
                                                 
{'loss': '0.5051', 'grad_norm': '0.25', 'learning_rate': '9.727e-06', 'ppl': '1.657', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6724', 'tokens/total': 13762560, 'tokens/trainable': 12826468, 'epoch': '0.4008'}

 20%|█▉        | 105/536 [09:16<16:34,  2.31s/it]
 20%|█▉        | 106/536 [09:19<16:28,  2.30s/it]
 20%|█▉        | 107/536 [09:21<16:26,  2.30s/it]
 20%|██        | 108/536 [09:23<16:27,  2.31s/it][2026-03-16 19:20:58,221] [INFO] [axolotl.core.trainers.base._save:721] [PID:213] Saving model checkpoint to ./outputs/qwen3-sft-stmt-tk/checkpoint-108


Writing model shards:   0%|          | 0/1 [00:00<?, ?it/s]

Writing model shards: 100%|██████████| 1/1 [00:17<00:00, 17.13s/it]
Writing model shards: 100%|██████████| 1/1 [00:17<00:00, 17.13s/it]

 20%|██        | 109/536 [11:03<3:44:22, 31.53s/it]
 21%|██        | 110/536 [11:05<2:41:43, 22.78s/it]
                                                   
{'loss': '0.4725', 'grad_norm': '0.2266', 'learning_rate': '9.672e-06', 'ppl': '1.604', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6506', 'tokens/total': 14417920, 'tokens/trainable': 13440042, 'epoch': '0.4198'}

 21%|██        | 110/536 [11:05<2:41:43, 22.78s/it]
 21%|██        | 111/536 [11:07<1:57:41, 16.61s/it]
 21%|██        | 112/536 [11:10<1:26:54, 12.30s/it]
 21%|██        | 113/536 [11:12<1:05:42,  9.32s/it]
 21%|██▏       | 114/536 [11:14<50:52,  7.23s/it]  
 21%|██▏       | 115/536 [11:17<40:26,  5.76s/it]
                                                 
{'loss': '0.5004', 'grad_norm': '0.2256', 'learning_rate': '9.612e-06', 'ppl': '1.649', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6525', 'tokens/total': 15073280, 'tokens/trainable': 14049913, 'epoch': '0.4389'}

 21%|██▏       | 115/536 [11:17<40:26,  5.76s/it]
 22%|██▏       | 116/536 [11:19<33:04,  4.72s/it]
 22%|██▏       | 117/536 [11:22<28:43,  4.11s/it]
 22%|██▏       | 118/536 [11:24<24:56,  3.58s/it]
 22%|██▏       | 119/536 [11:26<22:09,  3.19s/it]
 22%|██▏       | 120/536 [11:29<20:22,  2.94s/it]
                                                 
{'loss': '0.4727', 'grad_norm': '0.248', 'learning_rate': '9.546e-06', 'ppl': '1.604', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6422', 'tokens/total': 15728640, 'tokens/trainable': 14657396, 'epoch': '0.458'}

 22%|██▏       | 120/536 [11:29<20:22,  2.94s/it]
 23%|██▎       | 121/536 [11:31<19:02,  2.75s/it]
 23%|██▎       | 122/536 [11:33<18:01,  2.61s/it]
 23%|██▎       | 123/536 [11:36<17:25,  2.53s/it]
 23%|██▎       | 124/536 [11:38<17:02,  2.48s/it]
 23%|██▎       | 125/536 [11:40<16:31,  2.41s/it]
                                                 
{'loss': '0.4808', 'grad_norm': '0.2344', 'learning_rate': '9.476e-06', 'ppl': '1.617', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6830', 'tokens/total': 16384000, 'tokens/trainable': 15266794, 'epoch': '0.4771'}

 23%|██▎       | 125/536 [11:40<16:31,  2.41s/it]
 24%|██▎       | 126/536 [11:43<16:21,  2.39s/it]
 24%|██▎       | 127/536 [11:45<16:21,  2.40s/it]
 24%|██▍       | 128/536 [11:47<16:06,  2.37s/it]
 24%|██▍       | 129/536 [11:50<15:59,  2.36s/it]
 24%|██▍       | 130/536 [11:52<15:57,  2.36s/it]
                                                 
{'loss': '0.4726', 'grad_norm': '0.2451', 'learning_rate': '9.401e-06', 'ppl': '1.604', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6471', 'tokens/total': 17039360, 'tokens/trainable': 15876387, 'epoch': '0.4962'}

 24%|██▍       | 130/536 [11:52<15:57,  2.36s/it]
 24%|██▍       | 131/536 [11:54<15:48,  2.34s/it]
 25%|██▍       | 132/536 [11:57<15:37,  2.32s/it]
 25%|██▍       | 133/536 [11:59<15:31,  2.31s/it]
 25%|██▌       | 134/536 [12:01<15:54,  2.37s/it]
 25%|██▌       | 135/536 [12:04<16:09,  2.42s/it]
                                                 
{'loss': '0.4864', 'grad_norm': '0.2344', 'learning_rate': '9.322e-06', 'ppl': '1.626', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6056', 'tokens/total': 17694720, 'tokens/trainable': 16486440, 'epoch': '0.5153'}

 25%|██▌       | 135/536 [12:04<16:09,  2.42s/it][2026-03-16 19:23:38,988] [INFO] [axolotl.core.trainers.base._save:721] [PID:213] Saving model checkpoint to ./outputs/qwen3-sft-stmt-tk/checkpoint-135


Writing model shards:   0%|          | 0/1 [00:00<?, ?it/s]

Writing model shards: 100%|██████████| 1/1 [00:17<00:00, 17.41s/it]
Writing model shards: 100%|██████████| 1/1 [00:17<00:00, 17.41s/it]

 25%|██▌       | 136/536 [13:41<3:26:11, 30.93s/it]
 26%|██▌       | 137/536 [13:45<2:30:52, 22.69s/it]
 26%|██▌       | 138/536 [13:47<1:49:51, 16.56s/it]
 26%|██▌       | 139/536 [13:49<1:21:09, 12.27s/it]
 26%|██▌       | 140/536 [13:52<1:01:08,  9.26s/it]
                                                   
{'loss': '0.4817', 'grad_norm': '0.2275', 'learning_rate': '9.238e-06', 'ppl': '1.619', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6712', 'tokens/total': 18350080, 'tokens/trainable': 17095060, 'epoch': '0.5344'}

 26%|██▌       | 140/536 [13:52<1:01:08,  9.26s/it]
 26%|██▋       | 141/536 [13:54<47:09,  7.16s/it]  
 26%|██▋       | 142/536 [13:56<37:27,  5.70s/it]
 27%|██▋       | 143/536 [13:58<30:36,  4.67s/it]
 27%|██▋       | 144/536 [14:01<25:57,  3.97s/it]
 27%|██▋       | 145/536 [14:03<22:36,  3.47s/it]
                                                 
{'loss': '0.4827', 'grad_norm': '0.249', 'learning_rate': '9.149e-06', 'ppl': '1.62', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6633', 'tokens/total': 19005440, 'tokens/trainable': 17703368, 'epoch': '0.5534'}

 27%|██▋       | 145/536 [14:03<22:36,  3.47s/it]
 27%|██▋       | 146/536 [14:05<20:19,  3.13s/it]
 27%|██▋       | 147/536 [14:08<18:42,  2.89s/it]
 28%|██▊       | 148/536 [14:10<17:44,  2.74s/it]
 28%|██▊       | 149/536 [14:12<16:47,  2.60s/it]
 28%|██▊       | 150/536 [14:15<16:15,  2.53s/it]
                                                 
{'loss': '0.4892', 'grad_norm': '0.2217', 'learning_rate': '9.057e-06', 'ppl': '1.631', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6466', 'tokens/total': 19660800, 'tokens/trainable': 18311084, 'epoch': '0.5725'}

 28%|██▊       | 150/536 [14:15<16:15,  2.53s/it]
 28%|██▊       | 151/536 [14:17<15:49,  2.47s/it]
 28%|██▊       | 152/536 [14:20<15:51,  2.48s/it]
 29%|██▊       | 153/536 [14:22<16:18,  2.55s/it]
 29%|██▊       | 154/536 [14:25<16:18,  2.56s/it]
 29%|██▉       | 155/536 [14:27<16:04,  2.53s/it]
                                                 
{'loss': '0.4618', 'grad_norm': '0.2236', 'learning_rate': '8.959e-06', 'ppl': '1.587', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6104', 'tokens/total': 20316160, 'tokens/trainable': 18920000, 'epoch': '0.5916'}

 29%|██▉       | 155/536 [14:27<16:04,  2.53s/it]
 29%|██▉       | 156/536 [14:30<15:36,  2.47s/it]
 29%|██▉       | 157/536 [14:32<15:16,  2.42s/it]
 29%|██▉       | 158/536 [14:35<15:24,  2.45s/it]
 30%|██▉       | 159/536 [14:37<15:05,  2.40s/it]
 30%|██▉       | 160/536 [14:39<14:54,  2.38s/it]
                                                 
{'loss': '0.471', 'grad_norm': '0.2793', 'learning_rate': '8.858e-06', 'ppl': '1.602', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6516', 'tokens/total': 20971520, 'tokens/trainable': 19529720, 'epoch': '0.6107'}

 30%|██▉       | 160/536 [14:39<14:54,  2.38s/it]
 30%|███       | 161/536 [14:41<14:48,  2.37s/it]
 30%|███       | 162/536 [14:44<14:28,  2.32s/it][2026-03-16 19:26:18,649] [INFO] [axolotl.core.trainers.base._save:721] [PID:213] Saving model checkpoint to ./outputs/qwen3-sft-stmt-tk/checkpoint-162


Writing model shards:   0%|          | 0/1 [00:00<?, ?it/s]

Writing model shards: 100%|██████████| 1/1 [00:16<00:00, 16.63s/it]
Writing model shards: 100%|██████████| 1/1 [00:16<00:00, 16.63s/it]

 30%|███       | 163/536 [16:21<3:11:06, 30.74s/it]
 31%|███       | 164/536 [16:23<2:17:38, 22.20s/it]
 31%|███       | 165/536 [16:25<1:40:18, 16.22s/it]
                                                   
{'loss': '0.4703', 'grad_norm': '0.2383', 'learning_rate': '8.752e-06', 'ppl': '1.6', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6785', 'tokens/total': 21626880, 'tokens/trainable': 20137712, 'epoch': '0.6298'}

 31%|███       | 165/536 [16:25<1:40:18, 16.22s/it]
 31%|███       | 166/536 [16:28<1:14:13, 12.04s/it]
 31%|███       | 167/536 [16:30<56:04,  9.12s/it]  
 31%|███▏      | 168/536 [16:32<43:22,  7.07s/it]
 32%|███▏      | 169/536 [16:34<34:28,  5.64s/it]
 32%|███▏      | 170/536 [16:37<28:13,  4.63s/it]
                                                 
{'loss': '0.4727', 'grad_norm': '0.2139', 'learning_rate': '8.643e-06', 'ppl': '1.604', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6694', 'tokens/total': 22282240, 'tokens/trainable': 20749040, 'epoch': '0.6489'}

 32%|███▏      | 170/536 [16:37<28:13,  4.63s/it]
 32%|███▏      | 171/536 [16:39<24:07,  3.96s/it]
 32%|███▏      | 172/536 [16:42<21:19,  3.51s/it]
 32%|███▏      | 173/536 [16:44<19:34,  3.24s/it]
 32%|███▏      | 174/536 [16:46<17:46,  2.95s/it]
 33%|███▎      | 175/536 [16:49<16:24,  2.73s/it]
                                                 
{'loss': '0.4856', 'grad_norm': '0.2119', 'learning_rate': '8.53e-06', 'ppl': '1.625', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6812', 'tokens/total': 22937600, 'tokens/trainable': 21358216, 'epoch': '0.6679'}

 33%|███▎      | 175/536 [16:49<16:24,  2.73s/it]
 33%|███▎      | 176/536 [16:51<15:32,  2.59s/it]
 33%|███▎      | 177/536 [16:54<16:38,  2.78s/it]
 33%|███▎      | 178/536 [16:57<15:52,  2.66s/it]
 33%|███▎      | 179/536 [16:59<15:07,  2.54s/it]
 34%|███▎      | 180/536 [17:01<14:47,  2.49s/it]
                                                 
{'loss': '0.4551', 'grad_norm': '0.2266', 'learning_rate': '8.413e-06', 'ppl': '1.576', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6375', 'tokens/total': 23592960, 'tokens/trainable': 21963408, 'epoch': '0.687'}

 34%|███▎      | 180/536 [17:01<14:47,  2.49s/it]
 34%|███▍      | 181/536 [17:04<14:25,  2.44s/it]
 34%|███▍      | 182/536 [17:06<14:08,  2.40s/it]
 34%|███▍      | 183/536 [17:08<13:47,  2.34s/it]
 34%|███▍      | 184/536 [17:10<13:42,  2.34s/it]
 35%|███▍      | 185/536 [17:13<13:34,  2.32s/it]
                                                 
{'loss': '0.4654', 'grad_norm': '0.2695', 'learning_rate': '8.292e-06', 'ppl': '1.593', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6688', 'tokens/total': 24248320, 'tokens/trainable': 22570984, 'epoch': '0.7061'}

 35%|███▍      | 185/536 [17:13<13:34,  2.32s/it]
 35%|███▍      | 186/536 [17:15<13:21,  2.29s/it]
 35%|███▍      | 187/536 [17:17<13:19,  2.29s/it]
 35%|███▌      | 188/536 [17:19<13:11,  2.28s/it]
 35%|███▌      | 189/536 [17:22<13:08,  2.27s/it][2026-03-16 19:28:56,617] [INFO] [axolotl.core.trainers.base._save:721] [PID:213] Saving model checkpoint to ./outputs/qwen3-sft-stmt-tk/checkpoint-189


Writing model shards:   0%|          | 0/1 [00:00<?, ?it/s]

Writing model shards: 100%|██████████| 1/1 [00:17<00:00, 17.04s/it]
Writing model shards: 100%|██████████| 1/1 [00:17<00:00, 17.04s/it]

 35%|███▌      | 190/536 [19:01<3:00:49, 31.36s/it]
                                                   
{'loss': '0.4727', 'grad_norm': '0.2285', 'learning_rate': '8.168e-06', 'ppl': '1.604', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '4795', 'tokens/total': 24903680, 'tokens/trainable': 23180680, 'epoch': '0.7252'}

 35%|███▌      | 190/536 [19:01<3:00:49, 31.36s/it]
 36%|███▌      | 191/536 [19:03<2:10:15, 22.65s/it]
 36%|███▌      | 192/536 [19:06<1:35:23, 16.64s/it]
 36%|███▌      | 193/536 [19:08<1:11:08, 12.44s/it]
 36%|███▌      | 194/536 [19:11<53:42,  9.42s/it]  
 36%|███▋      | 195/536 [19:13<41:27,  7.30s/it]
                                                 
{'loss': '0.462', 'grad_norm': '0.2158', 'learning_rate': '8.041e-06', 'ppl': '1.587', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6567', 'tokens/total': 25559040, 'tokens/trainable': 23790730, 'epoch': '0.7443'}

 36%|███▋      | 195/536 [19:13<41:27,  7.30s/it]
 37%|███▋      | 196/536 [19:16<32:54,  5.81s/it]
 37%|███▋      | 197/536 [19:18<27:50,  4.93s/it]
 37%|███▋      | 198/536 [19:21<23:17,  4.13s/it]
 37%|███▋      | 199/536 [19:23<20:04,  3.57s/it]
 37%|███▋      | 200/536 [19:25<17:43,  3.17s/it]
                                                 
{'loss': '0.4676', 'grad_norm': '0.2188', 'learning_rate': '7.91e-06', 'ppl': '1.596', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6880', 'tokens/total': 26214400, 'tokens/trainable': 24401252, 'epoch': '0.7634'}

 37%|███▋      | 200/536 [19:25<17:43,  3.17s/it]
 38%|███▊      | 201/536 [19:27<16:11,  2.90s/it]
 38%|███▊      | 202/536 [19:30<15:05,  2.71s/it]
 38%|███▊      | 203/536 [19:32<14:15,  2.57s/it]
 38%|███▊      | 204/536 [19:34<13:41,  2.47s/it]
 38%|███▊      | 205/536 [19:37<13:22,  2.43s/it]
                                                 
{'loss': '0.4504', 'grad_norm': '0.2158', 'learning_rate': '7.776e-06', 'ppl': '1.569', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6617', 'tokens/total': 26869760, 'tokens/trainable': 25010696, 'epoch': '0.7824'}

 38%|███▊      | 205/536 [19:37<13:22,  2.43s/it]
 38%|███▊      | 206/536 [19:39<13:08,  2.39s/it]
 39%|███▊      | 207/536 [19:41<12:51,  2.34s/it]
 39%|███▉      | 208/536 [19:43<12:41,  2.32s/it]
 39%|███▉      | 209/536 [19:46<12:40,  2.32s/it]
 39%|███▉      | 210/536 [19:48<13:00,  2.39s/it]
                                                 
{'loss': '0.4614', 'grad_norm': '0.2295', 'learning_rate': '7.639e-06', 'ppl': '1.586', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '5994', 'tokens/total': 27525120, 'tokens/trainable': 25617872, 'epoch': '0.8015'}

 39%|███▉      | 210/536 [19:48<13:00,  2.39s/it]
 39%|███▉      | 211/536 [19:51<13:14,  2.44s/it]
 40%|███▉      | 212/536 [19:53<12:58,  2.40s/it]
 40%|███▉      | 213/536 [19:55<12:42,  2.36s/it]
 40%|███▉      | 214/536 [19:58<12:29,  2.33s/it]
 40%|████      | 215/536 [20:00<12:22,  2.31s/it]
                                                 
{'loss': '0.477', 'grad_norm': '0.2412', 'learning_rate': '7.5e-06', 'ppl': '1.611', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6697', 'tokens/total': 28180480, 'tokens/trainable': 26227438, 'epoch': '0.8206'}

 40%|████      | 215/536 [20:00<12:22,  2.31s/it]
 40%|████      | 216/536 [20:02<12:22,  2.32s/it][2026-03-16 19:31:37,309] [INFO] [axolotl.core.trainers.base._save:721] [PID:213] Saving model checkpoint to ./outputs/qwen3-sft-stmt-tk/checkpoint-216


Writing model shards:   0%|          | 0/1 [00:00<?, ?it/s]

Writing model shards: 100%|██████████| 1/1 [00:16<00:00, 16.85s/it]
Writing model shards: 100%|██████████| 1/1 [00:16<00:00, 16.85s/it]

 40%|████      | 217/536 [21:40<2:45:11, 31.07s/it]
 41%|████      | 218/536 [21:43<1:59:15, 22.50s/it]
 41%|████      | 219/536 [21:45<1:26:49, 16.43s/it]
 41%|████      | 220/536 [21:47<1:04:08, 12.18s/it]
                                                   
{'loss': '0.4535', 'grad_norm': '0.2148', 'learning_rate': '7.358e-06', 'ppl': '1.574', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6762', 'tokens/total': 28835840, 'tokens/trainable': 26833456, 'epoch': '0.8397'}

 41%|████      | 220/536 [21:47<1:04:08, 12.18s/it]
 41%|████      | 221/536 [21:50<48:25,  9.22s/it]  
 41%|████▏     | 222/536 [21:52<37:25,  7.15s/it]
 42%|████▏     | 223/536 [21:54<29:44,  5.70s/it]
 42%|████▏     | 224/536 [21:57<24:17,  4.67s/it]
 42%|████▏     | 225/536 [21:59<20:30,  3.96s/it]
                                                 
{'loss': '0.4639', 'grad_norm': '0.2197', 'learning_rate': '7.213e-06', 'ppl': '1.59', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6762', 'tokens/total': 29491200, 'tokens/trainable': 27444416, 'epoch': '0.8588'}

 42%|████▏     | 225/536 [21:59<20:30,  3.96s/it]
 42%|████▏     | 226/536 [22:01<17:50,  3.45s/it]
 42%|████▏     | 227/536 [22:03<15:56,  3.10s/it]
 43%|████▎     | 228/536 [22:06<14:40,  2.86s/it]
 43%|████▎     | 229/536 [22:08<14:24,  2.82s/it]
 43%|████▎     | 230/536 [22:11<14:01,  2.75s/it]
                                                 
{'loss': '0.4578', 'grad_norm': '0.2217', 'learning_rate': '7.066e-06', 'ppl': '1.581', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '5816', 'tokens/total': 30146560, 'tokens/trainable': 28048432, 'epoch': '0.8779'}

 43%|████▎     | 230/536 [22:11<14:01,  2.75s/it]
 43%|████▎     | 231/536 [22:13<13:21,  2.63s/it]
 43%|████▎     | 232/536 [22:16<12:45,  2.52s/it]
 43%|████▎     | 233/536 [22:18<12:29,  2.47s/it]
 44%|████▎     | 234/536 [22:20<12:15,  2.44s/it]
 44%|████▍     | 235/536 [22:23<11:59,  2.39s/it]
                                                 
{'loss': '0.4497', 'grad_norm': '0.2354', 'learning_rate': '6.917e-06', 'ppl': '1.568', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6580', 'tokens/total': 30801920, 'tokens/trainable': 28655952, 'epoch': '0.8969'}

 44%|████▍     | 235/536 [22:23<11:59,  2.39s/it]
 44%|████▍     | 236/536 [22:25<11:55,  2.39s/it]
 44%|████▍     | 237/536 [22:27<11:47,  2.37s/it]
 44%|████▍     | 238/536 [22:30<11:38,  2.34s/it]
 45%|████▍     | 239/536 [22:32<11:38,  2.35s/it]
 45%|████▍     | 240/536 [22:34<11:23,  2.31s/it]
                                                 
{'loss': '0.4693', 'grad_norm': '0.2275', 'learning_rate': '6.766e-06', 'ppl': '1.599', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6813', 'tokens/total': 31457280, 'tokens/trainable': 29262050, 'epoch': '0.916'}

 45%|████▍     | 240/536 [22:34<11:23,  2.31s/it]
 45%|████▍     | 241/536 [22:37<11:19,  2.30s/it]
 45%|████▌     | 242/536 [22:39<11:12,  2.29s/it]
 45%|████▌     | 243/536 [22:41<11:10,  2.29s/it][2026-03-16 19:34:16,197] [INFO] [axolotl.core.trainers.base._save:721] [PID:213] Saving model checkpoint to ./outputs/qwen3-sft-stmt-tk/checkpoint-243


Writing model shards:   0%|          | 0/1 [00:00<?, ?it/s]

Writing model shards: 100%|██████████| 1/1 [00:16<00:00, 16.81s/it]
Writing model shards: 100%|██████████| 1/1 [00:16<00:00, 16.82s/it]

 46%|████▌     | 244/536 [24:21<2:33:31, 31.55s/it]
 46%|████▌     | 245/536 [24:23<1:50:23, 22.76s/it]
                                                   
{'loss': '0.4629', 'grad_norm': '0.2178', 'learning_rate': '6.613e-06', 'ppl': '1.589', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6754', 'tokens/total': 32112640, 'tokens/trainable': 29868356, 'epoch': '0.9351'}

 46%|████▌     | 245/536 [24:23<1:50:23, 22.76s/it]
 46%|████▌     | 246/536 [24:25<1:20:19, 16.62s/it]
 46%|████▌     | 247/536 [24:28<59:24, 12.33s/it]  
 46%|████▋     | 248/536 [24:30<45:07,  9.40s/it]
 46%|████▋     | 249/536 [24:33<34:46,  7.27s/it]
 47%|████▋     | 250/536 [24:35<27:30,  5.77s/it]
                                                 
{'loss': '0.474', 'grad_norm': '0.2539', 'learning_rate': '6.458e-06', 'ppl': '1.606', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6714', 'tokens/total': 32768000, 'tokens/trainable': 30473100, 'epoch': '0.9542'}

 47%|████▋     | 250/536 [24:35<27:30,  5.77s/it]
 47%|████▋     | 251/536 [24:37<22:25,  4.72s/it]
 47%|████▋     | 252/536 [24:39<18:50,  3.98s/it]
 47%|████▋     | 253/536 [24:42<16:27,  3.49s/it]
 47%|████▋     | 254/536 [24:44<14:46,  3.14s/it]
 48%|████▊     | 255/536 [24:46<13:26,  2.87s/it]
                                                 
{'loss': '0.467', 'grad_norm': '0.2305', 'learning_rate': '6.302e-06', 'ppl': '1.595', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6772', 'tokens/total': 33423360, 'tokens/trainable': 31078478, 'epoch': '0.9733'}

 48%|████▊     | 255/536 [24:46<13:26,  2.87s/it]
 48%|████▊     | 256/536 [24:49<12:32,  2.69s/it]
 48%|████▊     | 257/536 [24:51<11:59,  2.58s/it]
 48%|████▊     | 258/536 [24:53<11:39,  2.52s/it]
 48%|████▊     | 259/536 [24:56<11:19,  2.45s/it]
 49%|████▊     | 260/536 [24:58<11:02,  2.40s/it]
                                                 
{'loss': '0.4511', 'grad_norm': '0.2148', 'learning_rate': '6.144e-06', 'ppl': '1.57', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6619', 'tokens/total': 34078720, 'tokens/trainable': 31682612, 'epoch': '0.9924'}

 49%|████▊     | 260/536 [24:58<11:02,  2.40s/it]
 49%|████▊     | 261/536 [25:00<10:50,  2.36s/it]
 49%|████▉     | 262/536 [25:03<10:56,  2.40s/it]
 49%|████▉     | 263/536 [25:06<12:20,  2.71s/it]
 49%|████▉     | 264/536 [25:08<11:41,  2.58s/it]
 49%|████▉     | 265/536 [25:11<11:22,  2.52s/it]
                                                 
{'loss': '0.4682', 'grad_norm': '0.2451', 'learning_rate': '5.985e-06', 'ppl': '1.597', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6472', 'tokens/total': 34734080, 'tokens/trainable': 32293470, 'epoch': '1.011'}

 49%|████▉     | 265/536 [25:11<11:22,  2.52s/it]
 50%|████▉     | 266/536 [25:13<11:14,  2.50s/it]
 50%|████▉     | 267/536 [25:16<11:28,  2.56s/it]
 50%|█████     | 268/536 [25:18<11:16,  2.52s/it]
 50%|█████     | 269/536 [25:21<10:53,  2.45s/it]
 50%|█████     | 270/536 [25:23<10:39,  2.40s/it]
                                                 
{'loss': '0.461', 'grad_norm': '0.2207', 'learning_rate': '5.826e-06', 'ppl': '1.586', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6685', 'tokens/total': 35389440, 'tokens/trainable': 32904464, 'epoch': '1.031'}

 50%|█████     | 270/536 [25:23<10:39,  2.40s/it][2026-03-16 19:36:59,256] [INFO] [axolotl.core.trainers.base._save:721] [PID:213] Saving model checkpoint to ./outputs/qwen3-sft-stmt-tk/checkpoint-270


Writing model shards:   0%|          | 0/1 [00:00<?, ?it/s]

Writing model shards: 100%|██████████| 1/1 [00:17<00:00, 17.30s/it]
Writing model shards: 100%|██████████| 1/1 [00:17<00:00, 17.30s/it]

 51%|█████     | 271/536 [27:02<2:18:44, 31.41s/it]
 51%|█████     | 272/536 [27:04<1:39:50, 22.69s/it]
 51%|█████     | 273/536 [27:07<1:12:35, 16.56s/it]
 51%|█████     | 274/536 [27:09<53:33, 12.26s/it]  
 51%|█████▏    | 275/536 [27:11<40:23,  9.29s/it]
                                                 
{'loss': '0.4545', 'grad_norm': '0.2324', 'learning_rate': '5.665e-06', 'ppl': '1.575', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6568', 'tokens/total': 36044800, 'tokens/trainable': 33517832, 'epoch': '1.05'}

 51%|█████▏    | 275/536 [27:11<40:23,  9.29s/it]
 51%|█████▏    | 276/536 [27:15<32:37,  7.53s/it]
 52%|█████▏    | 277/536 [27:17<25:45,  5.97s/it]
 52%|█████▏    | 278/536 [27:19<20:59,  4.88s/it]
 52%|█████▏    | 279/536 [27:22<17:35,  4.11s/it]
 52%|█████▏    | 280/536 [27:24<15:12,  3.56s/it]
                                                 
{'loss': '0.447', 'grad_norm': '0.2158', 'learning_rate': '5.503e-06', 'ppl': '1.564', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6754', 'tokens/total': 36700160, 'tokens/trainable': 34129632, 'epoch': '1.069'}

 52%|█████▏    | 280/536 [27:24<15:12,  3.56s/it]
 52%|█████▏    | 281/536 [27:26<13:35,  3.20s/it]
 53%|█████▎    | 282/536 [27:28<12:20,  2.91s/it]
 53%|█████▎    | 283/536 [27:31<11:30,  2.73s/it]
 53%|█████▎    | 284/536 [27:33<10:54,  2.60s/it]
 53%|█████▎    | 285/536 [27:36<10:51,  2.60s/it]
                                                 
{'loss': '0.4378', 'grad_norm': '0.2119', 'learning_rate': '5.341e-06', 'ppl': '1.549', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '5854', 'tokens/total': 37355520, 'tokens/trainable': 34742888, 'epoch': '1.088'}

 53%|█████▎    | 285/536 [27:36<10:51,  2.60s/it]
 53%|█████▎    | 286/536 [27:38<10:42,  2.57s/it]
 54%|█████▎    | 287/536 [27:41<10:26,  2.51s/it]
 54%|█████▎    | 288/536 [27:43<10:02,  2.43s/it]
 54%|█████▍    | 289/536 [27:45<09:45,  2.37s/it]
 54%|█████▍    | 290/536 [27:47<09:40,  2.36s/it]
                                                 
{'loss': '0.4756', 'grad_norm': '0.2246', 'learning_rate': '5.179e-06', 'ppl': '1.609', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6534', 'tokens/total': 38010880, 'tokens/trainable': 35352848, 'epoch': '1.107'}

 54%|█████▍    | 290/536 [27:47<09:40,  2.36s/it]
 54%|█████▍    | 291/536 [27:50<09:33,  2.34s/it]
 54%|█████▍    | 292/536 [27:52<09:27,  2.33s/it]
 55%|█████▍    | 293/536 [27:54<09:23,  2.32s/it]
 55%|█████▍    | 294/536 [27:57<09:23,  2.33s/it]
 55%|█████▌    | 295/536 [27:59<09:21,  2.33s/it]
                                                 
{'loss': '0.4635', 'grad_norm': '0.2188', 'learning_rate': '5.016e-06', 'ppl': '1.59', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6541', 'tokens/total': 38666240, 'tokens/trainable': 35964736, 'epoch': '1.126'}

 55%|█████▌    | 295/536 [27:59<09:21,  2.33s/it]
 55%|█████▌    | 296/536 [28:01<09:16,  2.32s/it]
 55%|█████▌    | 297/536 [28:03<09:09,  2.30s/it][2026-03-16 19:39:38,467] [INFO] [axolotl.core.trainers.base._save:721] [PID:213] Saving model checkpoint to ./outputs/qwen3-sft-stmt-tk/checkpoint-297


Writing model shards:   0%|          | 0/1 [00:00<?, ?it/s]

Writing model shards: 100%|██████████| 1/1 [00:16<00:00, 16.82s/it]
Writing model shards: 100%|██████████| 1/1 [00:16<00:00, 16.82s/it]

 56%|█████▌    | 298/536 [29:43<2:05:14, 31.57s/it]
 56%|█████▌    | 299/536 [29:46<1:30:06, 22.81s/it]
 56%|█████▌    | 300/536 [29:48<1:05:28, 16.65s/it]
                                                   
{'loss': '0.4578', 'grad_norm': '0.2334', 'learning_rate': '4.854e-06', 'ppl': '1.581', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6804', 'tokens/total': 39321600, 'tokens/trainable': 36579308, 'epoch': '1.145'}

 56%|█████▌    | 300/536 [29:48<1:05:28, 16.65s/it]
 56%|█████▌    | 301/536 [29:50<48:19, 12.34s/it]  
 56%|█████▋    | 302/536 [29:53<36:18,  9.31s/it]
 57%|█████▋    | 303/536 [29:55<27:59,  7.21s/it]
 57%|█████▋    | 304/536 [29:57<22:22,  5.78s/it]
 57%|█████▋    | 305/536 [30:00<18:50,  4.89s/it]
                                                 
{'loss': '0.4526', 'grad_norm': '0.2129', 'learning_rate': '4.691e-06', 'ppl': '1.572', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6214', 'tokens/total': 39976960, 'tokens/trainable': 37187912, 'epoch': '1.164'}

 57%|█████▋    | 305/536 [30:00<18:50,  4.89s/it]
 57%|█████▋    | 306/536 [30:02<15:44,  4.11s/it]
 57%|█████▋    | 307/536 [30:05<13:31,  3.55s/it]
 57%|█████▋    | 308/536 [30:07<12:06,  3.19s/it]
 58%|█████▊    | 309/536 [30:09<11:00,  2.91s/it]
 58%|█████▊    | 310/536 [30:12<10:18,  2.74s/it]
                                                 
{'loss': '0.4482', 'grad_norm': '0.21', 'learning_rate': '4.529e-06', 'ppl': '1.566', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6526', 'tokens/total': 40632320, 'tokens/trainable': 37799984, 'epoch': '1.183'}

 58%|█████▊    | 310/536 [30:12<10:18,  2.74s/it]
 58%|█████▊    | 311/536 [30:14<09:49,  2.62s/it]
 58%|█████▊    | 312/536 [30:16<09:25,  2.53s/it]
 58%|█████▊    | 313/536 [30:18<09:07,  2.45s/it]
 59%|█████▊    | 314/536 [30:21<09:33,  2.58s/it]
 59%|█████▉    | 315/536 [30:24<09:07,  2.48s/it]
                                                 
{'loss': '0.4544', 'grad_norm': '0.2148', 'learning_rate': '4.368e-06', 'ppl': '1.575', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6899', 'tokens/total': 41287680, 'tokens/trainable': 38409832, 'epoch': '1.202'}

 59%|█████▉    | 315/536 [30:24<09:07,  2.48s/it]
 59%|█████▉    | 316/536 [30:26<08:55,  2.43s/it]
 59%|█████▉    | 317/536 [30:28<08:44,  2.39s/it]
 59%|█████▉    | 318/536 [30:31<08:35,  2.36s/it]
 60%|█████▉    | 319/536 [30:33<08:25,  2.33s/it]
 60%|█████▉    | 320/536 [30:35<08:19,  2.31s/it]
                                                 
{'loss': '0.4539', 'grad_norm': '0.2285', 'learning_rate': '4.207e-06', 'ppl': '1.574', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6757', 'tokens/total': 41943040, 'tokens/trainable': 39020096, 'epoch': '1.221'}

 60%|█████▉    | 320/536 [30:35<08:19,  2.31s/it]
 60%|█████▉    | 321/536 [30:37<08:17,  2.31s/it]
 60%|██████    | 322/536 [30:40<08:09,  2.29s/it]
 60%|██████    | 323/536 [30:42<08:19,  2.35s/it]
 60%|██████    | 324/536 [30:45<08:26,  2.39s/it][2026-03-16 19:42:20,100] [INFO] [axolotl.core.trainers.base._save:721] [PID:213] Saving model checkpoint to ./outputs/qwen3-sft-stmt-tk/checkpoint-324


Writing model shards:   0%|          | 0/1 [00:00<?, ?it/s]

Writing model shards: 100%|██████████| 1/1 [00:16<00:00, 16.49s/it]
Writing model shards: 100%|██████████| 1/1 [00:16<00:00, 16.49s/it]

 61%|██████    | 325/536 [32:22<1:48:54, 30.97s/it]
                                                   
{'loss': '0.4481', 'grad_norm': '0.2246', 'learning_rate': '4.046e-06', 'ppl': '1.565', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6887', 'tokens/total': 42598400, 'tokens/trainable': 39629160, 'epoch': '1.24'}

 61%|██████    | 325/536 [32:22<1:48:54, 30.97s/it]
 61%|██████    | 326/536 [32:24<1:18:13, 22.35s/it]
 61%|██████    | 327/536 [32:27<56:54, 16.34s/it]  
 61%|██████    | 328/536 [32:29<42:00, 12.12s/it]
 61%|██████▏   | 329/536 [32:31<31:43,  9.20s/it]
 62%|██████▏   | 330/536 [32:34<24:27,  7.12s/it]
                                                 
{'loss': '0.4542', 'grad_norm': '0.2256', 'learning_rate': '3.887e-06', 'ppl': '1.575', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6731', 'tokens/total': 43253760, 'tokens/trainable': 40237288, 'epoch': '1.26'}

 62%|██████▏   | 330/536 [32:34<24:27,  7.12s/it]
 62%|██████▏   | 331/536 [32:36<19:21,  5.67s/it]
 62%|██████▏   | 332/536 [32:39<16:57,  4.99s/it]
 62%|██████▏   | 333/536 [32:42<14:06,  4.17s/it]
 62%|██████▏   | 334/536 [32:44<12:07,  3.60s/it]
 62%|██████▎   | 335/536 [32:46<10:43,  3.20s/it]
                                                 
{'loss': '0.4412', 'grad_norm': '0.2539', 'learning_rate': '3.729e-06', 'ppl': '1.555', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6745', 'tokens/total': 43909120, 'tokens/trainable': 40848032, 'epoch': '1.279'}

 62%|██████▎   | 335/536 [32:46<10:43,  3.20s/it]
 63%|██████▎   | 336/536 [32:48<09:46,  2.93s/it]
 63%|██████▎   | 337/536 [32:51<09:05,  2.74s/it]
 63%|██████▎   | 338/536 [32:53<08:37,  2.61s/it]
 63%|██████▎   | 339/536 [32:55<08:14,  2.51s/it]
 63%|██████▎   | 340/536 [32:58<08:04,  2.47s/it]
                                                 
{'loss': '0.4615', 'grad_norm': '0.2217', 'learning_rate': '3.573e-06', 'ppl': '1.586', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6370', 'tokens/total': 44564480, 'tokens/trainable': 41457624, 'epoch': '1.298'}

 63%|██████▎   | 340/536 [32:58<08:04,  2.47s/it]
 64%|██████▎   | 341/536 [33:00<08:05,  2.49s/it]
 64%|██████▍   | 342/536 [33:03<07:51,  2.43s/it]
 64%|██████▍   | 343/536 [33:05<07:49,  2.43s/it]
 64%|██████▍   | 344/536 [33:08<07:51,  2.46s/it]
 64%|██████▍   | 345/536 [33:10<07:38,  2.40s/it]
                                                 
{'loss': '0.4599', 'grad_norm': '0.2188', 'learning_rate': '3.418e-06', 'ppl': '1.584', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6795', 'tokens/total': 45219840, 'tokens/trainable': 42069272, 'epoch': '1.317'}

 64%|██████▍   | 345/536 [33:10<07:38,  2.40s/it]
 65%|██████▍   | 346/536 [33:12<07:29,  2.37s/it]
 65%|██████▍   | 347/536 [33:14<07:26,  2.36s/it]
 65%|██████▍   | 348/536 [33:17<07:20,  2.34s/it]
 65%|██████▌   | 349/536 [33:19<07:12,  2.32s/it]
 65%|██████▌   | 350/536 [33:21<07:09,  2.31s/it]
                                                 
{'loss': '0.4499', 'grad_norm': '0.2148', 'learning_rate': '3.264e-06', 'ppl': '1.568', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6691', 'tokens/total': 45875200, 'tokens/trainable': 42681132, 'epoch': '1.336'}

 65%|██████▌   | 350/536 [33:21<07:09,  2.31s/it]
 65%|██████▌   | 351/536 [33:24<07:04,  2.29s/it][2026-03-16 19:44:58,459] [INFO] [axolotl.core.trainers.base._save:721] [PID:213] Saving model checkpoint to ./outputs/qwen3-sft-stmt-tk/checkpoint-351


Writing model shards:   0%|          | 0/1 [00:00<?, ?it/s]

Writing model shards: 100%|██████████| 1/1 [00:16<00:00, 16.55s/it]
Writing model shards: 100%|██████████| 1/1 [00:16<00:00, 16.55s/it]

 66%|██████▌   | 352/536 [35:02<1:35:17, 31.07s/it]
 66%|██████▌   | 353/536 [35:04<1:08:31, 22.47s/it]
 66%|██████▌   | 354/536 [35:06<49:47, 16.42s/it]  
 66%|██████▌   | 355/536 [35:09<36:49, 12.21s/it]
                                                 
{'loss': '0.4529', 'grad_norm': '0.249', 'learning_rate': '3.113e-06', 'ppl': '1.573', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6371', 'tokens/total': 46530560, 'tokens/trainable': 43292904, 'epoch': '1.355'}

 66%|██████▌   | 355/536 [35:09<36:49, 12.21s/it]
 66%|██████▋   | 356/536 [35:11<27:37,  9.21s/it]
 67%|██████▋   | 357/536 [35:13<21:14,  7.12s/it]
 67%|██████▋   | 358/536 [35:16<16:48,  5.67s/it]
 67%|██████▋   | 359/536 [35:18<13:56,  4.73s/it]
 67%|██████▋   | 360/536 [35:21<11:51,  4.04s/it]
                                                 
{'loss': '0.4461', 'grad_norm': '0.2207', 'learning_rate': '2.963e-06', 'ppl': '1.562', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6150', 'tokens/total': 47185920, 'tokens/trainable': 43900272, 'epoch': '1.374'}

 67%|██████▋   | 360/536 [35:21<11:51,  4.04s/it]
 67%|██████▋   | 361/536 [35:23<10:14,  3.51s/it]
 68%|██████▊   | 362/536 [35:25<09:24,  3.25s/it]
 68%|██████▊   | 363/536 [35:28<08:33,  2.97s/it]
 68%|██████▊   | 364/536 [35:30<07:52,  2.75s/it]
 68%|██████▊   | 365/536 [35:32<07:23,  2.59s/it]
                                                 
{'loss': '0.4581', 'grad_norm': '0.3555', 'learning_rate': '2.816e-06', 'ppl': '1.581', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6767', 'tokens/total': 47841280, 'tokens/trainable': 44509872, 'epoch': '1.393'}

 68%|██████▊   | 365/536 [35:32<07:23,  2.59s/it]
 68%|██████▊   | 366/536 [35:34<07:04,  2.50s/it]
 68%|██████▊   | 367/536 [35:37<06:49,  2.42s/it]
 69%|██████▊   | 368/536 [35:39<06:39,  2.38s/it]
 69%|██████▉   | 369/536 [35:41<06:33,  2.36s/it]
 69%|██████▉   | 370/536 [35:44<06:33,  2.37s/it]
                                                 
{'loss': '0.4483', 'grad_norm': '0.2109', 'learning_rate': '2.671e-06', 'ppl': '1.566', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6359', 'tokens/total': 48496640, 'tokens/trainable': 45121444, 'epoch': '1.412'}

 69%|██████▉   | 370/536 [35:44<06:33,  2.37s/it]
 69%|██████▉   | 371/536 [35:46<06:27,  2.35s/it]
 69%|██████▉   | 372/536 [35:48<06:25,  2.35s/it]
 70%|██████▉   | 373/536 [35:51<06:17,  2.31s/it]
 70%|██████▉   | 374/536 [35:53<06:12,  2.30s/it]
 70%|██████▉   | 375/536 [35:55<06:13,  2.32s/it]
                                                 
{'loss': '0.4475', 'grad_norm': '0.2617', 'learning_rate': '2.528e-06', 'ppl': '1.564', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6505', 'tokens/total': 49152000, 'tokens/trainable': 45733296, 'epoch': '1.431'}

 70%|██████▉   | 375/536 [35:55<06:13,  2.32s/it]
 70%|███████   | 376/536 [35:58<06:09,  2.31s/it]
 70%|███████   | 377/536 [36:00<06:26,  2.43s/it]
 71%|███████   | 378/536 [36:03<06:17,  2.39s/it][2026-03-16 19:47:37,290] [INFO] [axolotl.core.trainers.base._save:721] [PID:213] Saving model checkpoint to ./outputs/qwen3-sft-stmt-tk/checkpoint-378


Writing model shards:   0%|          | 0/1 [00:00<?, ?it/s]

Writing model shards: 100%|██████████| 1/1 [00:16<00:00, 16.77s/it]
Writing model shards: 100%|██████████| 1/1 [00:16<00:00, 16.77s/it]

 71%|███████   | 379/536 [37:39<1:20:25, 30.73s/it]
 71%|███████   | 380/536 [37:42<57:44, 22.21s/it]  
                                                 
{'loss': '0.4467', 'grad_norm': '0.208', 'learning_rate': '2.388e-06', 'ppl': '1.563', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6639', 'tokens/total': 49807360, 'tokens/trainable': 46341868, 'epoch': '1.45'}

 71%|███████   | 380/536 [37:42<57:44, 22.21s/it]
 71%|███████   | 381/536 [37:44<41:54, 16.22s/it]
 71%|███████▏  | 382/536 [37:46<31:01, 12.09s/it]
 71%|███████▏  | 383/536 [37:49<23:20,  9.16s/it]
 72%|███████▏  | 384/536 [37:51<17:57,  7.09s/it]
 72%|███████▏  | 385/536 [37:53<14:15,  5.66s/it]
                                                 
{'loss': '0.4373', 'grad_norm': '0.2129', 'learning_rate': '2.251e-06', 'ppl': '1.549', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6487', 'tokens/total': 50462720, 'tokens/trainable': 46948144, 'epoch': '1.469'}

 72%|███████▏  | 385/536 [37:53<14:15,  5.66s/it]
 72%|███████▏  | 386/536 [37:56<11:39,  4.67s/it]
 72%|███████▏  | 387/536 [37:58<09:46,  3.94s/it]
 72%|███████▏  | 388/536 [38:00<08:31,  3.45s/it]
 73%|███████▎  | 389/536 [38:03<07:40,  3.14s/it]
 73%|███████▎  | 390/536 [38:05<07:00,  2.88s/it]
                                                 
{'loss': '0.452', 'grad_norm': '0.2314', 'learning_rate': '2.117e-06', 'ppl': '1.571', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6670', 'tokens/total': 51118080, 'tokens/trainable': 47558056, 'epoch': '1.489'}

 73%|███████▎  | 390/536 [38:05<07:00,  2.88s/it]
 73%|███████▎  | 391/536 [38:08<07:22,  3.05s/it]
 73%|███████▎  | 392/536 [38:11<06:49,  2.85s/it]
 73%|███████▎  | 393/536 [38:13<06:22,  2.68s/it]
 74%|███████▎  | 394/536 [38:15<06:02,  2.55s/it]
 74%|███████▎  | 395/536 [38:18<06:01,  2.56s/it]
                                                 
{'loss': '0.4435', 'grad_norm': '0.2139', 'learning_rate': '1.985e-06', 'ppl': '1.558', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '5877', 'tokens/total': 51773440, 'tokens/trainable': 48168520, 'epoch': '1.508'}

 74%|███████▎  | 395/536 [38:18<06:01,  2.56s/it]
 74%|███████▍  | 396/536 [38:20<05:47,  2.48s/it]
 74%|███████▍  | 397/536 [38:22<05:35,  2.42s/it]
 74%|███████▍  | 398/536 [38:25<05:27,  2.37s/it]
 74%|███████▍  | 399/536 [38:27<05:22,  2.35s/it]
 75%|███████▍  | 400/536 [38:29<05:21,  2.37s/it]
                                                 
{'loss': '0.4444', 'grad_norm': '0.2236', 'learning_rate': '1.857e-06', 'ppl': '1.56', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6406', 'tokens/total': 52428800, 'tokens/trainable': 48779416, 'epoch': '1.527'}

 75%|███████▍  | 400/536 [38:29<05:21,  2.37s/it]
 75%|███████▍  | 401/536 [38:32<05:15,  2.34s/it]
 75%|███████▌  | 402/536 [38:34<05:10,  2.32s/it]
 75%|███████▌  | 403/536 [38:36<05:10,  2.34s/it]
 75%|███████▌  | 404/536 [38:39<05:07,  2.33s/it]
 76%|███████▌  | 405/536 [38:41<05:03,  2.32s/it]
                                                 
{'loss': '0.4557', 'grad_norm': '0.2324', 'learning_rate': '1.732e-06', 'ppl': '1.577', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6719', 'tokens/total': 53084160, 'tokens/trainable': 49387032, 'epoch': '1.546'}

 76%|███████▌  | 405/536 [38:41<05:03,  2.32s/it][2026-03-16 19:50:15,755] [INFO] [axolotl.core.trainers.base._save:721] [PID:213] Saving model checkpoint to ./outputs/qwen3-sft-stmt-tk/checkpoint-405


Writing model shards:   0%|          | 0/1 [00:00<?, ?it/s]

Writing model shards: 100%|██████████| 1/1 [00:16<00:00, 16.46s/it]
Writing model shards: 100%|██████████| 1/1 [00:16<00:00, 16.46s/it]

 76%|███████▌  | 406/536 [40:19<1:06:58, 30.91s/it]
 76%|███████▌  | 407/536 [40:21<48:00, 22.33s/it]  
 76%|███████▌  | 408/536 [40:23<34:49, 16.33s/it]
 76%|███████▋  | 409/536 [40:25<25:39, 12.12s/it]
 76%|███████▋  | 410/536 [40:28<19:38,  9.36s/it]
                                                 
{'loss': '0.4617', 'grad_norm': '0.2168', 'learning_rate': '1.611e-06', 'ppl': '1.587', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '5250', 'tokens/total': 53739520, 'tokens/trainable': 49995344, 'epoch': '1.565'}

 76%|███████▋  | 410/536 [40:28<19:38,  9.36s/it]
 77%|███████▋  | 411/536 [40:31<15:03,  7.23s/it]
 77%|███████▋  | 412/536 [40:33<11:54,  5.76s/it]
 77%|███████▋  | 413/536 [40:36<09:53,  4.83s/it]
 77%|███████▋  | 414/536 [40:38<08:16,  4.07s/it]
 77%|███████▋  | 415/536 [40:40<07:07,  3.53s/it]
                                                 
{'loss': '0.4492', 'grad_norm': '0.2217', 'learning_rate': '1.493e-06', 'ppl': '1.567', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6671', 'tokens/total': 54394880, 'tokens/trainable': 50603264, 'epoch': '1.584'}

 77%|███████▋  | 415/536 [40:40<07:07,  3.53s/it]
 78%|███████▊  | 416/536 [40:43<06:19,  3.17s/it]
 78%|███████▊  | 417/536 [40:45<05:45,  2.90s/it]
 78%|███████▊  | 418/536 [40:47<05:26,  2.77s/it]
 78%|███████▊  | 419/536 [40:50<05:18,  2.73s/it]
 78%|███████▊  | 420/536 [40:52<05:02,  2.61s/it]
                                                 
{'loss': '0.4522', 'grad_norm': '0.2676', 'learning_rate': '1.379e-06', 'ppl': '1.572', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6557', 'tokens/total': 55050240, 'tokens/trainable': 51213612, 'epoch': '1.603'}

 78%|███████▊  | 420/536 [40:52<05:02,  2.61s/it]
 79%|███████▊  | 421/536 [40:55<04:49,  2.52s/it]
 79%|███████▊  | 422/536 [40:57<04:40,  2.46s/it]
 79%|███████▉  | 423/536 [40:59<04:40,  2.49s/it]
 79%|███████▉  | 424/536 [41:02<04:29,  2.41s/it]
 79%|███████▉  | 425/536 [41:04<04:21,  2.36s/it]
                                                 
{'loss': '0.4414', 'grad_norm': '0.2168', 'learning_rate': '1.269e-06', 'ppl': '1.555', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6755', 'tokens/total': 55705600, 'tokens/trainable': 51819592, 'epoch': '1.622'}

 79%|███████▉  | 425/536 [41:04<04:21,  2.36s/it]
 79%|███████▉  | 426/536 [41:06<04:17,  2.34s/it]
 80%|███████▉  | 427/536 [41:08<04:13,  2.32s/it]
 80%|███████▉  | 428/536 [41:11<04:08,  2.30s/it]
 80%|████████  | 429/536 [41:13<04:07,  2.31s/it]
 80%|████████  | 430/536 [41:15<04:04,  2.31s/it]
                                                 
{'loss': '0.4532', 'grad_norm': '0.2217', 'learning_rate': '1.163e-06', 'ppl': '1.573', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6642', 'tokens/total': 56360960, 'tokens/trainable': 52431520, 'epoch': '1.641'}

 80%|████████  | 430/536 [41:15<04:04,  2.31s/it]
 80%|████████  | 431/536 [41:18<04:10,  2.38s/it]
 81%|████████  | 432/536 [41:20<04:04,  2.35s/it][2026-03-16 19:52:55,057] [INFO] [axolotl.core.trainers.base._save:721] [PID:213] Saving model checkpoint to ./outputs/qwen3-sft-stmt-tk/checkpoint-432


Writing model shards:   0%|          | 0/1 [00:00<?, ?it/s]

Writing model shards: 100%|██████████| 1/1 [00:16<00:00, 16.45s/it]
Writing model shards: 100%|██████████| 1/1 [00:16<00:00, 16.45s/it]

 81%|████████  | 433/536 [42:58<53:14, 31.01s/it]
 81%|████████  | 434/536 [43:00<38:05, 22.41s/it]
 81%|████████  | 435/536 [43:03<27:33, 16.37s/it]
                                                 
{'loss': '0.4605', 'grad_norm': '0.3574', 'learning_rate': '1.061e-06', 'ppl': '1.585', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6726', 'tokens/total': 57016320, 'tokens/trainable': 53041944, 'epoch': '1.66'}

 81%|████████  | 435/536 [43:03<27:33, 16.37s/it]
 81%|████████▏ | 436/536 [43:05<20:20, 12.20s/it]
 82%|████████▏ | 437/536 [43:07<15:11,  9.20s/it]
 82%|████████▏ | 438/536 [43:10<11:41,  7.16s/it]
 82%|████████▏ | 439/536 [43:12<09:11,  5.68s/it]
 82%|████████▏ | 440/536 [43:14<07:29,  4.68s/it]
                                                 
{'loss': '0.446', 'grad_norm': '0.2119', 'learning_rate': '9.626e-07', 'ppl': '1.562', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6513', 'tokens/total': 57671680, 'tokens/trainable': 53647180, 'epoch': '1.679'}

 82%|████████▏ | 440/536 [43:14<07:29,  4.68s/it]
 82%|████████▏ | 441/536 [43:17<06:23,  4.04s/it]
 82%|████████▏ | 442/536 [43:19<05:31,  3.53s/it]
 83%|████████▎ | 443/536 [43:21<04:53,  3.16s/it]
 83%|████████▎ | 444/536 [43:24<04:26,  2.90s/it]
 83%|████████▎ | 445/536 [43:26<04:04,  2.69s/it]
                                                 
{'loss': '0.4299', 'grad_norm': '0.2188', 'learning_rate': '8.688e-07', 'ppl': '1.537', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6951', 'tokens/total': 58327040, 'tokens/trainable': 54256432, 'epoch': '1.698'}

 83%|████████▎ | 445/536 [43:26<04:04,  2.69s/it]
 83%|████████▎ | 446/536 [43:28<03:52,  2.58s/it]
 83%|████████▎ | 447/536 [43:31<03:41,  2.49s/it]
 84%|████████▎ | 448/536 [43:33<03:32,  2.41s/it]
 84%|████████▍ | 449/536 [43:35<03:30,  2.42s/it]
 84%|████████▍ | 450/536 [43:38<03:23,  2.37s/it]
                                                 
{'loss': '0.4583', 'grad_norm': '0.2188', 'learning_rate': '7.794e-07', 'ppl': '1.581', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6777', 'tokens/total': 58982400, 'tokens/trainable': 54863148, 'epoch': '1.718'}

 84%|████████▍ | 450/536 [43:38<03:23,  2.37s/it]
 84%|████████▍ | 451/536 [43:40<03:28,  2.46s/it]
 84%|████████▍ | 452/536 [43:43<03:24,  2.44s/it]
 85%|████████▍ | 453/536 [43:45<03:19,  2.40s/it]
 85%|████████▍ | 454/536 [43:47<03:15,  2.38s/it]
 85%|████████▍ | 455/536 [43:50<03:12,  2.38s/it]
                                                 
{'loss': '0.4523', 'grad_norm': '0.2119', 'learning_rate': '6.945e-07', 'ppl': '1.572', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6454', 'tokens/total': 59637760, 'tokens/trainable': 55471580, 'epoch': '1.737'}

 85%|████████▍ | 455/536 [43:50<03:12,  2.38s/it]
 85%|████████▌ | 456/536 [43:52<03:15,  2.44s/it]
 85%|████████▌ | 457/536 [43:54<03:10,  2.41s/it]
 85%|████████▌ | 458/536 [43:57<03:11,  2.46s/it]
 86%|████████▌ | 459/536 [43:59<03:05,  2.41s/it][2026-03-16 19:55:34,637] [INFO] [axolotl.core.trainers.base._save:721] [PID:213] Saving model checkpoint to ./outputs/qwen3-sft-stmt-tk/checkpoint-459


Writing model shards:   0%|          | 0/1 [00:00<?, ?it/s]

Writing model shards: 100%|██████████| 1/1 [00:16<00:00, 16.63s/it]
Writing model shards: 100%|██████████| 1/1 [00:16<00:00, 16.63s/it]

 86%|████████▌ | 460/536 [45:37<39:11, 30.94s/it]
                                                 
{'loss': '0.4523', 'grad_norm': '0.2148', 'learning_rate': '6.141e-07', 'ppl': '1.572', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6711', 'tokens/total': 60293120, 'tokens/trainable': 56081832, 'epoch': '1.756'}

 86%|████████▌ | 460/536 [45:37<39:11, 30.94s/it]
 86%|████████▌ | 461/536 [45:39<27:55, 22.34s/it]
 86%|████████▌ | 462/536 [45:41<20:06, 16.30s/it]
 86%|████████▋ | 463/536 [45:44<14:42, 12.09s/it]
 87%|████████▋ | 464/536 [45:46<10:58,  9.14s/it]
 87%|████████▋ | 465/536 [45:48<08:21,  7.06s/it]
                                                 
{'loss': '0.4461', 'grad_norm': '0.4551', 'learning_rate': '5.383e-07', 'ppl': '1.562', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6903', 'tokens/total': 60948480, 'tokens/trainable': 56694240, 'epoch': '1.775'}

 87%|████████▋ | 465/536 [45:48<08:21,  7.06s/it]
 87%|████████▋ | 466/536 [45:50<06:33,  5.62s/it]
 87%|████████▋ | 467/536 [45:53<05:23,  4.69s/it]
 87%|████████▋ | 468/536 [45:55<04:29,  3.97s/it]
 88%|████████▊ | 469/536 [45:57<03:50,  3.44s/it]
 88%|████████▊ | 470/536 [46:01<03:45,  3.42s/it]
                                                 
{'loss': '0.4341', 'grad_norm': '0.208', 'learning_rate': '4.673e-07', 'ppl': '1.544', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '4512', 'tokens/total': 61603840, 'tokens/trainable': 57301404, 'epoch': '1.794'}

 88%|████████▊ | 470/536 [46:01<03:45,  3.42s/it]
 88%|████████▊ | 471/536 [46:03<03:20,  3.09s/it]
 88%|████████▊ | 472/536 [46:05<03:02,  2.86s/it]
 88%|████████▊ | 473/536 [46:08<02:50,  2.70s/it]
 88%|████████▊ | 474/536 [46:10<02:39,  2.58s/it]
 89%|████████▊ | 475/536 [46:12<02:34,  2.54s/it]
                                                 
{'loss': '0.4627', 'grad_norm': '0.2461', 'learning_rate': '4.011e-07', 'ppl': '1.588', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6249', 'tokens/total': 62259200, 'tokens/trainable': 57910216, 'epoch': '1.813'}

 89%|████████▊ | 475/536 [46:12<02:34,  2.54s/it]
 89%|████████▉ | 476/536 [46:15<02:26,  2.45s/it]
 89%|████████▉ | 477/536 [46:17<02:24,  2.45s/it]
 89%|████████▉ | 478/536 [46:20<02:25,  2.51s/it]
 89%|████████▉ | 479/536 [46:22<02:19,  2.45s/it]
 90%|████████▉ | 480/536 [46:24<02:13,  2.39s/it]
                                                 
{'loss': '0.4538', 'grad_norm': '0.2178', 'learning_rate': '3.397e-07', 'ppl': '1.574', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6635', 'tokens/total': 62914560, 'tokens/trainable': 58517712, 'epoch': '1.832'}

 90%|████████▉ | 480/536 [46:24<02:13,  2.39s/it]
 90%|████████▉ | 481/536 [46:27<02:09,  2.36s/it]
 90%|████████▉ | 482/536 [46:29<02:05,  2.33s/it]
 90%|█████████ | 483/536 [46:31<02:03,  2.33s/it]
 90%|█████████ | 484/536 [46:34<02:01,  2.33s/it]
 90%|█████████ | 485/536 [46:36<02:04,  2.44s/it]
                                                 
{'loss': '0.4395', 'grad_norm': '0.208', 'learning_rate': '2.833e-07', 'ppl': '1.552', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6504', 'tokens/total': 63569920, 'tokens/trainable': 59125608, 'epoch': '1.851'}

 90%|█████████ | 485/536 [46:36<02:04,  2.44s/it]
 91%|█████████ | 486/536 [46:39<01:59,  2.39s/it][2026-03-16 19:58:13,418] [INFO] [axolotl.core.trainers.base._save:721] [PID:213] Saving model checkpoint to ./outputs/qwen3-sft-stmt-tk/checkpoint-486


Writing model shards:   0%|          | 0/1 [00:00<?, ?it/s]

Writing model shards: 100%|██████████| 1/1 [00:16<00:00, 16.82s/it]
Writing model shards: 100%|██████████| 1/1 [00:16<00:00, 16.82s/it]

 91%|█████████ | 487/536 [48:16<25:09, 30.81s/it]
 91%|█████████ | 488/536 [48:18<17:48, 22.26s/it]
 91%|█████████ | 489/536 [48:20<12:44, 16.26s/it]
 91%|█████████▏| 490/536 [48:24<09:32, 12.45s/it]
                                                 
{'loss': '0.4478', 'grad_norm': '0.2236', 'learning_rate': '2.318e-07', 'ppl': '1.565', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '4290', 'tokens/total': 64225280, 'tokens/trainable': 59733412, 'epoch': '1.87'}

 91%|█████████▏| 490/536 [48:24<09:32, 12.45s/it]
 92%|█████████▏| 491/536 [48:26<07:02,  9.39s/it]
 92%|█████████▏| 492/536 [48:28<05:19,  7.26s/it]
 92%|█████████▏| 493/536 [48:31<04:08,  5.78s/it]
 92%|█████████▏| 494/536 [48:33<03:22,  4.82s/it]
 92%|█████████▏| 495/536 [48:36<02:47,  4.08s/it]
                                                 
{'loss': '0.4362', 'grad_norm': '0.2129', 'learning_rate': '1.854e-07', 'ppl': '1.547', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6524', 'tokens/total': 64880640, 'tokens/trainable': 60339904, 'epoch': '1.889'}

 92%|█████████▏| 495/536 [48:36<02:47,  4.08s/it]
 93%|█████████▎| 496/536 [48:38<02:25,  3.64s/it]
 93%|█████████▎| 497/536 [48:40<02:06,  3.23s/it]
 93%|█████████▎| 498/536 [48:43<01:52,  2.96s/it]
 93%|█████████▎| 499/536 [48:45<01:42,  2.76s/it]
 93%|█████████▎| 500/536 [48:47<01:34,  2.62s/it]
                                                 
{'loss': '0.4656', 'grad_norm': '0.2217', 'learning_rate': '1.441e-07', 'ppl': '1.593', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6649', 'tokens/total': 65536000, 'tokens/trainable': 60945412, 'epoch': '1.908'}

 93%|█████████▎| 500/536 [48:47<01:34,  2.62s/it]
 93%|█████████▎| 501/536 [48:50<01:29,  2.55s/it]
 94%|█████████▎| 502/536 [48:52<01:25,  2.52s/it]
 94%|█████████▍| 503/536 [48:54<01:20,  2.44s/it]
 94%|█████████▍| 504/536 [48:57<01:16,  2.39s/it]
 94%|█████████▍| 505/536 [48:59<01:13,  2.36s/it]
                                                 
{'loss': '0.4466', 'grad_norm': '0.2129', 'learning_rate': '1.079e-07', 'ppl': '1.563', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6658', 'tokens/total': 66191360, 'tokens/trainable': 61550936, 'epoch': '1.927'}

 94%|█████████▍| 505/536 [48:59<01:13,  2.36s/it]
 94%|█████████▍| 506/536 [49:01<01:09,  2.33s/it]
 95%|█████████▍| 507/536 [49:04<01:07,  2.31s/it]
 95%|█████████▍| 508/536 [49:06<01:04,  2.31s/it]
 95%|█████████▍| 509/536 [49:08<01:04,  2.37s/it]
 95%|█████████▌| 510/536 [49:11<01:01,  2.35s/it]
                                                 
{'loss': '0.4754', 'grad_norm': '0.2168', 'learning_rate': '7.691e-08', 'ppl': '1.609', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6670', 'tokens/total': 66846720, 'tokens/trainable': 62157088, 'epoch': '1.947'}

 95%|█████████▌| 510/536 [49:11<01:01,  2.35s/it]
 95%|█████████▌| 511/536 [49:13<00:58,  2.34s/it]
 96%|█████████▌| 512/536 [49:15<00:55,  2.32s/it]
 96%|█████████▌| 513/536 [49:18<00:53,  2.31s/it][2026-03-16 20:00:52,618] [INFO] [axolotl.core.trainers.base._save:721] [PID:213] Saving model checkpoint to ./outputs/qwen3-sft-stmt-tk/checkpoint-513


Writing model shards:   0%|          | 0/1 [00:00<?, ?it/s]

Writing model shards: 100%|██████████| 1/1 [00:16<00:00, 16.79s/it]
Writing model shards: 100%|██████████| 1/1 [00:16<00:00, 16.79s/it]

 96%|█████████▌| 514/536 [50:57<11:33, 31.50s/it]
 96%|█████████▌| 515/536 [51:00<07:58, 22.77s/it]
                                                 
{'loss': '0.4548', 'grad_norm': '0.2217', 'learning_rate': '5.11e-08', 'ppl': '1.576', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6342', 'tokens/total': 67502080, 'tokens/trainable': 62762592, 'epoch': '1.966'}

 96%|█████████▌| 515/536 [51:00<07:58, 22.77s/it]
 96%|█████████▋| 516/536 [51:02<05:32, 16.65s/it]
 96%|█████████▋| 517/536 [51:04<03:54, 12.32s/it]
 97%|█████████▋| 518/536 [51:06<02:47,  9.30s/it]
 97%|█████████▋| 519/536 [51:09<02:02,  7.18s/it]
 97%|█████████▋| 520/536 [51:11<01:32,  5.76s/it]
                                                 
{'loss': '0.4544', 'grad_norm': '0.2314', 'learning_rate': '3.054e-08', 'ppl': '1.575', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6125', 'tokens/total': 68157440, 'tokens/trainable': 63366800, 'epoch': '1.985'}

 97%|█████████▋| 520/536 [51:11<01:32,  5.76s/it]
 97%|█████████▋| 521/536 [51:13<01:10,  4.72s/it]
 97%|█████████▋| 522/536 [51:16<00:55,  3.98s/it]
 98%|█████████▊| 523/536 [51:18<00:45,  3.47s/it]
 98%|█████████▊| 524/536 [51:20<00:38,  3.19s/it]
 98%|█████████▊| 525/536 [51:24<00:35,  3.21s/it]
                                                 
{'loss': '0.4504', 'grad_norm': '0.2246', 'learning_rate': '1.522e-08', 'ppl': '1.569', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6753', 'tokens/total': 68812800, 'tokens/trainable': 63975056, 'epoch': '2.004'}

 98%|█████████▊| 525/536 [51:24<00:35,  3.21s/it]
 98%|█████████▊| 526/536 [51:26<00:29,  2.92s/it]
 98%|█████████▊| 527/536 [51:28<00:24,  2.75s/it]
 99%|█████████▊| 528/536 [51:31<00:21,  2.71s/it]
 99%|█████████▊| 529/536 [51:33<00:18,  2.60s/it]
 99%|█████████▉| 530/536 [51:36<00:15,  2.52s/it]
                                                 
{'loss': '0.4546', 'grad_norm': '0.2109', 'learning_rate': '5.182e-09', 'ppl': '1.576', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6575', 'tokens/total': 69468160, 'tokens/trainable': 64586448, 'epoch': '2.023'}

 99%|█████████▉| 530/536 [51:36<00:15,  2.52s/it]
 99%|█████████▉| 531/536 [51:38<00:12,  2.45s/it]
 99%|█████████▉| 532/536 [51:40<00:09,  2.40s/it]
 99%|█████████▉| 533/536 [51:43<00:07,  2.40s/it]
100%|█████████▉| 534/536 [51:45<00:04,  2.39s/it]
100%|█████████▉| 535/536 [51:47<00:02,  2.42s/it]
                                                 
{'loss': '0.4493', 'grad_norm': '0.2314', 'learning_rate': '4.231e-10', 'ppl': '1.567', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'tokens/train_per_sec_per_gpu': '6115', 'tokens/total': 70123520, 'tokens/trainable': 65199364, 'epoch': '2.042'}

100%|█████████▉| 535/536 [51:47<00:02,  2.42s/it]
100%|██████████| 536/536 [51:50<00:00,  2.37s/it][2026-03-16 20:03:25,941] [INFO] [axolotl.core.trainers.base._save:721] [PID:213] Saving model checkpoint to ./outputs/qwen3-sft-stmt-tk/checkpoint-536


Writing model shards:   0%|          | 0/1 [00:00<?, ?it/s]

Writing model shards: 100%|██████████| 1/1 [00:16<00:00, 16.68s/it]
Writing model shards: 100%|██████████| 1/1 [00:16<00:00, 16.68s/it]

                                                 
{'train_runtime': '3210', 'train_samples_per_second': '2.672', 'train_steps_per_second': '0.167', 'train_loss': '0.4897', 'memory/max_active (GiB)': '22.63', 'memory/max_allocated (GiB)': '22.63', 'memory/device_reserved (GiB)': '29.23', 'epoch': '2.046', 'tokens/train_per_sec_per_gpu': '6757'}

100%|██████████| 536/536 [53:26<00:00,  2.37s/it]
100%|██████████| 536/536 [53:26<00:00,  5.98s/it]
[2026-03-16 20:04:52,263] [INFO] [axolotl.train.save_trained_model:237] [PID:213] Training completed! Saving trained model to ./outputs/qwen3-sft-stmt-tk/.
[2026-03-16 20:05:01,009] [INFO] [axolotl.core.trainers.base._save:721] [PID:213] Saving model checkpoint to ./outputs/qwen3-sft-stmt-tk/

Writing model shards:   0%|          | 0/1 [00:00<?, ?it/s]
Writing model shards: 100%|██████████| 1/1 [00:17<00:00, 17.53s/it]
Writing model shards: 100%|██████████| 1/1 [00:17<00:00, 17.53s/it]
[2026-03-16 20:05:19,091] [INFO] [axolotl.train.save_trained_model:351] [PID:213] Model successfully saved to ./outputs/qwen3-sft-stmt-tk/