[2025-11-06 16:11:41,516] [DEBUG] [axolotl.utils.config.log_gpu_memory_usage:127] [PID:837642] baseline 0.000GB ()
[2025-11-06 16:11:41,516] [INFO] [axolotl.cli.config.load_cfg:248] [PID:837642] config:
{
  "activation_offloading": false,
  "axolotl_config_path": "3b-qat-nvfp4.yaml",
  "base_model": "meta-llama/Llama-3.2-3B",
  "base_model_config": "meta-llama/Llama-3.2-3B",
  "batch_size": 64,
  "bf16": true,
  "capabilities": {
    "bf16": true,
    "compute_capability": "sm_90",
    "fp8": false,
    "n_gpu": 1,
    "n_node": 1
  },
  "context_parallel_size": 1,
  "cosine_constant_lr_ratio": 0.0,
  "cosine_min_lr_ratio": 1.0,
  "dataloader_num_workers": 1,
  "dataloader_pin_memory": true,
  "dataloader_prefetch_factor": 256,
  "dataset_prepared_path": "./outputs/dataset_prepared",
  "dataset_processes": 128,
  "datasets": [
    {
      "message_property_mappings": {
        "content": "content",
        "role": "role"
      },
      "path": "yahma/alpaca-cleaned",
      "split": "train[:95%]",
      "trust_remote_code": false,
      "type": "alpaca"
    }
  ],
  "ddp": false,
  "device": "cuda:0",
  "dion_rank_fraction": 1.0,
  "dion_rank_multiple_of": 1,
  "env_capabilities": {
    "torch_version": "2.8.0"
  },
  "eval_batch_size": 64,
  "eval_causal_lm_metrics": [
    "sacrebleu",
    "comet",
    "ter",
    "chrf"
  ],
  "eval_max_new_tokens": 128,
  "eval_table_size": 0,
  "evals_per_epoch": 1,
  "experimental_skip_move_to_device": true,
  "flash_attention": true,
  "fp16": false,
  "gradient_accumulation_steps": 1,
  "gradient_checkpointing": true,
  "gradient_checkpointing_kwargs": {
    "use_reentrant": true
  },
  "hub_model_id": "AlexHung29629/3b-qat-nvfp4",
  "include_tkps": true,
  "is_llama_derived_model": true,
  "learning_rate": 2e-05,
  "liger_fused_linear_cross_entropy": true,
  "liger_glu_activation": true,
  "liger_layer_norm": true,
  "liger_rms_norm": true,
  "liger_rope": true,
  "lisa_layers_attribute": "model.layers",
  "load_best_model_at_end": false,
  "load_in_4bit": false,
  "load_in_8bit": false,
  "local_rank": 0,
  "logging_steps": 1,
  "lora_dropout": 0.0,
  "loraplus_lr_embedding": 1e-06,
  "lr_scheduler": "cosine",
  "mean_resizing_embeddings": false,
  "micro_batch_size": 64,
  "model_config_type": "llama",
  "num_epochs": 1.0,
  "optimizer": "adamw_torch_fused",
  "output_dir": "./outputs/qat_out/",
  "plugins": [
    "axolotl.integrations.liger.LigerPlugin"
  ],
  "pretrain_multipack_attn": true,
  "profiler_steps_start": 0,
  "qat": {
    "activation_dtype": "TorchAOQuantDType.nvfp4",
    "group_size": 16,
    "quantize_embedding": false,
    "weight_dtype": "TorchAOQuantDType.nvfp4"
  },
  "qlora_sharded_model_loading": false,
  "ray_num_workers": 1,
  "resources_per_worker": {
    "GPU": 1
  },
  "sample_packing_bin_size": 200,
  "sample_packing_group_size": 100000,
  "save_first_step": true,
  "save_only_model": true,
  "save_safetensors": true,
  "saves_per_epoch": 1,
  "sequence_len": 8192,
  "shuffle_before_merging_datasets": false,
  "shuffle_merged_datasets": true,
  "skip_prepare_dataset": false,
  "special_tokens": {
    "pad_token": "<|finetune_right_pad_id|>"
  },
  "streaming_multipack_buffer_size": 10000,
  "strict": false,
  "tensor_parallel_size": 1,
  "tiled_mlp_use_original_mlp": true,
  "tokenizer_config": "meta-llama/Llama-3.2-3B",
  "tokenizer_save_jinja_files": true,
  "torch_dtype": "torch.bfloat16",
  "train_on_inputs": false,
  "trl": {
    "log_completions": false,
    "mask_truncated_completions": false,
    "ref_model_mixup_alpha": 0.9,
    "ref_model_sync_steps": 64,
    "scale_rewards": true,
    "sync_ref_model": false,
    "use_vllm": false,
    "vllm_server_host": "0.0.0.0",
    "vllm_server_port": 8000
  },
  "use_ray": false,
  "val_set_size": 0.0,
  "vllm": {
    "device": "auto",
    "dtype": "auto",
    "gpu_memory_utilization": 0.9,
    "host": "0.0.0.0",
    "port": 8000
  },
  "warmup_ratio": 0.1,
  "weight_decay": 0.0,
  "world_size": 1
}
[2025-11-06 16:11:46,489] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:278] [PID:837642] EOS: 128001 / <|end_of_text|>
[2025-11-06 16:11:46,489] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:279] [PID:837642] BOS: 128000 / <|begin_of_text|>
[2025-11-06 16:11:46,489] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:280] [PID:837642] PAD: 128004 / <|finetune_right_pad_id|>
[2025-11-06 16:11:46,489] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:281] [PID:837642] UNK: None / None
[2025-11-06 16:11:46,489] [INFO] [axolotl.loaders.tokenizer.load_tokenizer:295] [PID:837642] No Chat template selected. Consider adding a chat template for easier inference.
[2025-11-06 16:11:46,492] [INFO] [axolotl.utils.data.shared.load_preprocessed_dataset:476] [PID:837642] Unable to find prepared dataset in outputs/dataset_prepared/9bc662aed65b76546b2d635b3957a343
[2025-11-06 16:11:46,492] [INFO] [axolotl.utils.data.sft._load_raw_datasets:320] [PID:837642] Loading raw datasets...
[2025-11-06 16:11:46,492] [WARNING] [axolotl.utils.data.sft._load_raw_datasets:322] [PID:837642] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`.

Generating train split:   0%|          | 0/51760 [00:00<?, ? examples/s]
Generating train split: 100%|██████████| 51760/51760 [00:00<00:00, 229845.07 examples/s]
Generating train split: 100%|██████████| 51760/51760 [00:00<00:00, 228708.96 examples/s]
[2025-11-06 16:12:08,982] [INFO] [axolotl.utils.data.wrappers.get_dataset_wrapper:87] [PID:837642] Loading dataset: yahma/alpaca-cleaned with base_type: alpaca and prompt_style: None

Tokenizing Prompts (num_proc=128):   0%|          | 0/49172 [00:00<?, ? examples/s]
Tokenizing Prompts (num_proc=128):   0%|          | 84/49172 [00:01<13:38, 59.94 examples/s]
Tokenizing Prompts (num_proc=128):   0%|          | 176/49172 [00:01<06:01, 135.38 examples/s]
Tokenizing Prompts (num_proc=128):   2%|▏         | 792/49172 [00:01<01:05, 743.93 examples/s]
Tokenizing Prompts (num_proc=128):   3%|▎         | 1289/49172 [00:01<00:39, 1224.84 examples/s]
Tokenizing Prompts (num_proc=128):   3%|▎         | 1663/49172 [00:01<00:30, 1550.01 examples/s]
Tokenizing Prompts (num_proc=128):   4%|▍         | 2053/49172 [00:02<00:25, 1838.37 examples/s]
Tokenizing Prompts (num_proc=128):   5%|▌         | 2462/49172 [00:02<00:22, 2120.82 examples/s]
Tokenizing Prompts (num_proc=128):   6%|▌         | 2931/49172 [00:02<00:17, 2593.78 examples/s]
Tokenizing Prompts (num_proc=128):   7%|▋         | 3316/49172 [00:02<00:17, 2658.78 examples/s]
Tokenizing Prompts (num_proc=128):   8%|▊         | 3704/49172 [00:02<00:16, 2738.71 examples/s]
Tokenizing Prompts (num_proc=128):   8%|▊         | 4072/49172 [00:02<00:16, 2765.32 examples/s]
Tokenizing Prompts (num_proc=128):   9%|▉         | 4392/49172 [00:02<00:16, 2672.66 examples/s]
Tokenizing Prompts (num_proc=128):  10%|▉         | 4715/49172 [00:03<00:17, 2538.79 examples/s]
Tokenizing Prompts (num_proc=128):  10%|█         | 5094/49172 [00:03<00:17, 2586.15 examples/s]
Tokenizing Prompts (num_proc=128):  11%|█         | 5504/49172 [00:03<00:16, 2692.82 examples/s]
Tokenizing Prompts (num_proc=128):  12%|█▏        | 5910/49172 [00:03<00:15, 2763.20 examples/s]
Tokenizing Prompts (num_proc=128):  13%|█▎        | 6311/49172 [00:03<00:14, 2975.07 examples/s]
Tokenizing Prompts (num_proc=128):  14%|█▎        | 6668/49172 [00:03<00:14, 2859.49 examples/s]
Tokenizing Prompts (num_proc=128):  14%|█▍        | 7058/49172 [00:03<00:14, 2844.49 examples/s]
Tokenizing Prompts (num_proc=128):  15%|█▌        | 7385/49172 [00:03<00:15, 2728.62 examples/s]
Tokenizing Prompts (num_proc=128):  16%|█▌        | 7836/49172 [00:04<00:13, 3015.74 examples/s]
Tokenizing Prompts (num_proc=128):  17%|█▋        | 8193/49172 [00:04<00:14, 2873.13 examples/s]
Tokenizing Prompts (num_proc=128):  17%|█▋        | 8537/49172 [00:04<00:14, 2768.24 examples/s]
Tokenizing Prompts (num_proc=128):  18%|█▊        | 8959/49172 [00:04<00:13, 2885.47 examples/s]
Tokenizing Prompts (num_proc=128):  19%|█▉        | 9339/49172 [00:04<00:14, 2805.07 examples/s]
Tokenizing Prompts (num_proc=128):  20%|█▉        | 9755/49172 [00:04<00:13, 2846.53 examples/s]
Tokenizing Prompts (num_proc=128):  21%|██        | 10145/49172 [00:04<00:13, 2839.00 examples/s]
Tokenizing Prompts (num_proc=128):  22%|██▏       | 10628/49172 [00:05<00:12, 3040.97 examples/s]
Tokenizing Prompts (num_proc=128):  22%|██▏       | 11039/49172 [00:05<00:11, 3253.40 examples/s]
Tokenizing Prompts (num_proc=128):  23%|██▎       | 11385/49172 [00:05<00:12, 2982.44 examples/s]
Tokenizing Prompts (num_proc=128):  24%|██▍       | 11692/49172 [00:05<00:13, 2756.63 examples/s]
Tokenizing Prompts (num_proc=128):  25%|██▍       | 12063/49172 [00:05<00:13, 2720.60 examples/s]
Tokenizing Prompts (num_proc=128):  25%|██▌       | 12463/49172 [00:05<00:13, 2810.95 examples/s]
Tokenizing Prompts (num_proc=128):  26%|██▌       | 12850/49172 [00:05<00:12, 2819.15 examples/s]
Tokenizing Prompts (num_proc=128):  27%|██▋       | 13292/49172 [00:06<00:12, 2922.32 examples/s]
Tokenizing Prompts (num_proc=128):  28%|██▊       | 13600/49172 [00:06<00:13, 2700.65 examples/s]
Tokenizing Prompts (num_proc=128):  28%|██▊       | 13989/49172 [00:06<00:12, 2716.86 examples/s]
Tokenizing Prompts (num_proc=128):  29%|██▉       | 14475/49172 [00:06<00:12, 2852.98 examples/s]
Tokenizing Prompts (num_proc=128):  30%|███       | 14914/49172 [00:06<00:11, 2938.70 examples/s]
Tokenizing Prompts (num_proc=128):  31%|███▏      | 15412/49172 [00:06<00:10, 3223.11 examples/s]
Tokenizing Prompts (num_proc=128):  32%|███▏      | 15748/49172 [00:06<00:11, 3025.74 examples/s]
Tokenizing Prompts (num_proc=128):  33%|███▎      | 16102/49172 [00:06<00:11, 2858.09 examples/s]
Tokenizing Prompts (num_proc=128):  33%|███▎      | 16438/49172 [00:07<00:12, 2720.40 examples/s]
Tokenizing Prompts (num_proc=128):  34%|███▍      | 16767/49172 [00:07<00:12, 2595.58 examples/s]
Tokenizing Prompts (num_proc=128):  35%|███▍      | 17149/49172 [00:07<00:12, 2665.96 examples/s]
Tokenizing Prompts (num_proc=128):  36%|███▌      | 17516/49172 [00:07<00:11, 2643.92 examples/s]
Tokenizing Prompts (num_proc=128):  37%|███▋      | 17961/49172 [00:07<00:10, 2849.24 examples/s]
Tokenizing Prompts (num_proc=128):  37%|███▋      | 18279/49172 [00:07<00:10, 2822.57 examples/s]
Tokenizing Prompts (num_proc=128):  38%|███▊      | 18574/49172 [00:07<00:11, 2591.21 examples/s]
Tokenizing Prompts (num_proc=128):  39%|███▊      | 18957/49172 [00:08<00:11, 2625.56 examples/s]
Tokenizing Prompts (num_proc=128):  40%|███▉      | 19438/49172 [00:08<00:10, 2894.61 examples/s]
Tokenizing Prompts (num_proc=128):  40%|████      | 19776/49172 [00:08<00:10, 2741.27 examples/s]
Tokenizing Prompts (num_proc=128):  41%|████      | 20133/49172 [00:08<00:10, 2676.71 examples/s]
Tokenizing Prompts (num_proc=128):  42%|████▏     | 20582/49172 [00:08<00:10, 2813.67 examples/s]
Tokenizing Prompts (num_proc=128):  43%|████▎     | 21056/49172 [00:08<00:09, 2991.88 examples/s]
Tokenizing Prompts (num_proc=128):  44%|████▎     | 21507/49172 [00:08<00:08, 3120.60 examples/s]
Tokenizing Prompts (num_proc=128):  44%|████▍     | 21844/49172 [00:09<00:09, 2927.37 examples/s]
Tokenizing Prompts (num_proc=128):  45%|████▌     | 22214/49172 [00:09<00:09, 2980.95 examples/s]
Tokenizing Prompts (num_proc=128):  46%|████▌     | 22524/49172 [00:09<00:09, 2740.32 examples/s]
Tokenizing Prompts (num_proc=128):  46%|████▋     | 22828/49172 [00:09<00:09, 2639.81 examples/s]
Tokenizing Prompts (num_proc=128):  47%|████▋     | 23146/49172 [00:09<00:10, 2510.35 examples/s]
Tokenizing Prompts (num_proc=128):  48%|████▊     | 23619/49172 [00:09<00:09, 2763.04 examples/s]
Tokenizing Prompts (num_proc=128):  49%|████▉     | 24069/49172 [00:09<00:08, 2964.79 examples/s]
Tokenizing Prompts (num_proc=128):  50%|████▉     | 24455/49172 [00:09<00:08, 2924.79 examples/s]
Tokenizing Prompts (num_proc=128):  50%|█████     | 24809/49172 [00:10<00:08, 2856.24 examples/s]
Tokenizing Prompts (num_proc=128):  51%|█████     | 25112/49172 [00:10<00:09, 2621.31 examples/s]
Tokenizing Prompts (num_proc=128):  52%|█████▏    | 25522/49172 [00:10<00:08, 2714.48 examples/s]
Tokenizing Prompts (num_proc=128):  53%|█████▎    | 25873/49172 [00:10<00:08, 2705.01 examples/s]
Tokenizing Prompts (num_proc=128):  54%|█████▎    | 26322/49172 [00:10<00:07, 2907.63 examples/s]
Tokenizing Prompts (num_proc=128):  54%|█████▍    | 26671/49172 [00:10<00:08, 2787.83 examples/s]
Tokenizing Prompts (num_proc=128):  55%|█████▌    | 27127/49172 [00:10<00:07, 2923.69 examples/s]
Tokenizing Prompts (num_proc=128):  56%|█████▌    | 27506/49172 [00:11<00:07, 2929.98 examples/s]
Tokenizing Prompts (num_proc=128):  57%|█████▋    | 27928/49172 [00:11<00:07, 2972.71 examples/s]
Tokenizing Prompts (num_proc=128):  58%|█████▊    | 28285/49172 [00:11<00:06, 3017.41 examples/s]
Tokenizing Prompts (num_proc=128):  58%|█████▊    | 28589/49172 [00:11<00:07, 2769.13 examples/s]
Tokenizing Prompts (num_proc=128):  59%|█████▉    | 28960/49172 [00:11<00:07, 2746.25 examples/s]
Tokenizing Prompts (num_proc=128):  60%|█████▉    | 29343/49172 [00:11<00:07, 2735.06 examples/s]
Tokenizing Prompts (num_proc=128):  60%|██████    | 29730/49172 [00:11<00:07, 2777.41 examples/s]
Tokenizing Prompts (num_proc=128):  61%|██████▏   | 30201/49172 [00:11<00:06, 3027.86 examples/s]
Tokenizing Prompts (num_proc=128):  62%|██████▏   | 30511/49172 [00:12<00:06, 2794.02 examples/s]
Tokenizing Prompts (num_proc=128):  63%|██████▎   | 30963/49172 [00:12<00:06, 2860.65 examples/s]
Tokenizing Prompts (num_proc=128):  64%|██████▎   | 31260/49172 [00:12<00:06, 2659.00 examples/s]
Tokenizing Prompts (num_proc=128):  64%|██████▍   | 31661/49172 [00:12<00:06, 2744.06 examples/s]
Tokenizing Prompts (num_proc=128):  65%|██████▌   | 32103/49172 [00:12<00:05, 2886.04 examples/s]
Tokenizing Prompts (num_proc=128):  66%|██████▌   | 32523/49172 [00:12<00:05, 2982.93 examples/s]
Tokenizing Prompts (num_proc=128):  67%|██████▋   | 32894/49172 [00:12<00:05, 2895.08 examples/s]
Tokenizing Prompts (num_proc=128):  68%|██████▊   | 33277/49172 [00:13<00:05, 3037.84 examples/s]
Tokenizing Prompts (num_proc=128):  68%|██████▊   | 33649/49172 [00:13<00:05, 2842.83 examples/s]
Tokenizing Prompts (num_proc=128):  69%|██████▉   | 33971/49172 [00:13<00:05, 2690.27 examples/s]
Tokenizing Prompts (num_proc=128):  70%|██████▉   | 34337/49172 [00:13<00:05, 2695.46 examples/s]
Tokenizing Prompts (num_proc=128):  71%|███████   | 34730/49172 [00:13<00:05, 2719.08 examples/s]
Tokenizing Prompts (num_proc=128):  72%|███████▏  | 35193/49172 [00:13<00:04, 2910.87 examples/s]
Tokenizing Prompts (num_proc=128):  72%|███████▏  | 35583/49172 [00:13<00:04, 2876.12 examples/s]
Tokenizing Prompts (num_proc=128):  73%|███████▎  | 35965/49172 [00:14<00:04, 2847.46 examples/s]
Tokenizing Prompts (num_proc=128):  74%|███████▍  | 36341/49172 [00:14<00:04, 2829.40 examples/s]
Tokenizing Prompts (num_proc=128):  75%|███████▍  | 36647/49172 [00:14<00:04, 2673.80 examples/s]
Tokenizing Prompts (num_proc=128):  75%|███████▌  | 37103/49172 [00:14<00:04, 2839.03 examples/s]
Tokenizing Prompts (num_proc=128):  76%|███████▋  | 37568/49172 [00:14<00:03, 3009.77 examples/s]
Tokenizing Prompts (num_proc=128):  77%|███████▋  | 37894/49172 [00:14<00:03, 2833.19 examples/s]
Tokenizing Prompts (num_proc=128):  78%|███████▊  | 38273/49172 [00:14<00:03, 2797.05 examples/s]
Tokenizing Prompts (num_proc=128):  79%|███████▊  | 38625/49172 [00:15<00:03, 2713.00 examples/s]
Tokenizing Prompts (num_proc=128):  80%|███████▉  | 39134/49172 [00:15<00:03, 2989.24 examples/s]
Tokenizing Prompts (num_proc=128):  80%|████████  | 39484/49172 [00:15<00:03, 2970.93 examples/s]
Tokenizing Prompts (num_proc=128):  81%|████████  | 39830/49172 [00:15<00:03, 2822.53 examples/s]
Tokenizing Prompts (num_proc=128):  82%|████████▏ | 40145/49172 [00:15<00:03, 2691.61 examples/s]
Tokenizing Prompts (num_proc=128):  83%|████████▎ | 40611/49172 [00:15<00:02, 2919.81 examples/s]
Tokenizing Prompts (num_proc=128):  83%|████████▎ | 40933/49172 [00:15<00:02, 2872.22 examples/s]
Tokenizing Prompts (num_proc=128):  84%|████████▍ | 41235/49172 [00:15<00:03, 2550.35 examples/s]
Tokenizing Prompts (num_proc=128):  85%|████████▍ | 41636/49172 [00:16<00:02, 2623.37 examples/s]
Tokenizing Prompts (num_proc=128):  85%|████████▌ | 42011/49172 [00:16<00:02, 2667.31 examples/s]
Tokenizing Prompts (num_proc=128):  87%|████████▋ | 42565/49172 [00:16<00:02, 3151.00 examples/s]
Tokenizing Prompts (num_proc=128):  87%|████████▋ | 42957/49172 [00:16<00:02, 3026.85 examples/s]
Tokenizing Prompts (num_proc=128):  88%|████████▊ | 43325/49172 [00:16<00:02, 2828.92 examples/s]
Tokenizing Prompts (num_proc=128):  89%|████████▉ | 43700/49172 [00:16<00:01, 2829.86 examples/s]
Tokenizing Prompts (num_proc=128):  90%|████████▉ | 44022/49172 [00:16<00:01, 2649.38 examples/s]
Tokenizing Prompts (num_proc=128):  90%|█████████ | 44422/49172 [00:17<00:01, 2713.96 examples/s]
Tokenizing Prompts (num_proc=128):  91%|█████████ | 44782/49172 [00:17<00:01, 2703.83 examples/s]
Tokenizing Prompts (num_proc=128):  92%|█████████▏| 45155/49172 [00:17<00:01, 2707.69 examples/s]
Tokenizing Prompts (num_proc=128):  93%|█████████▎| 45628/49172 [00:17<00:01, 2900.02 examples/s]
Tokenizing Prompts (num_proc=128):  94%|█████████▎| 45986/49172 [00:17<00:01, 2826.26 examples/s]
Tokenizing Prompts (num_proc=128):  94%|█████████▍| 46333/49172 [00:17<00:01, 2738.12 examples/s]
Tokenizing Prompts (num_proc=128):  95%|█████████▌| 46787/49172 [00:17<00:00, 3126.02 examples/s]
Tokenizing Prompts (num_proc=128):  96%|█████████▌| 47266/49172 [00:17<00:00, 3503.55 examples/s]
Tokenizing Prompts (num_proc=128):  97%|█████████▋| 47643/49172 [00:18<00:00, 3338.62 examples/s]
Tokenizing Prompts (num_proc=128):  98%|█████████▊| 48031/49172 [00:18<00:00, 3013.54 examples/s]
Tokenizing Prompts (num_proc=128):  99%|█████████▊| 48485/49172 [00:18<00:00, 3381.58 examples/s]
Tokenizing Prompts (num_proc=128):  99%|█████████▉| 48913/49172 [00:18<00:00, 3410.78 examples/s]
Tokenizing Prompts (num_proc=128): 100%|██████████| 49172/49172 [00:19<00:00, 2564.69 examples/s]
[2025-11-06 16:12:28,393] [INFO] [axolotl.utils.data.utils.handle_long_seq_in_dataset:218] [PID:837642] min_input_len: 33
[2025-11-06 16:12:28,393] [INFO] [axolotl.utils.data.utils.handle_long_seq_in_dataset:220] [PID:837642] max_input_len: 1051

Dropping Long Sequences (>8192) (num_proc=128):   0%|          | 0/49172 [00:00<?, ? examples/s]
Dropping Long Sequences (>8192) (num_proc=128):   1%|          | 385/49172 [00:00<01:43, 471.65 examples/s]
Dropping Long Sequences (>8192) (num_proc=128):   5%|▌         | 2695/49172 [00:00<00:12, 3750.47 examples/s]
Dropping Long Sequences (>8192) (num_proc=128):  13%|█▎        | 6545/49172 [00:01<00:04, 9405.16 examples/s]
Dropping Long Sequences (>8192) (num_proc=128):  19%|█▉        | 9236/49172 [00:01<00:03, 12478.76 examples/s]
Dropping Long Sequences (>8192) (num_proc=128):  25%|██▌       | 12308/49172 [00:01<00:02, 15931.93 examples/s]
Dropping Long Sequences (>8192) (num_proc=128):  30%|███       | 14996/49172 [00:01<00:01, 17930.73 examples/s]
Dropping Long Sequences (>8192) (num_proc=128):  37%|███▋      | 18068/49172 [00:01<00:01, 20677.88 examples/s]
Dropping Long Sequences (>8192) (num_proc=128):  44%|████▍     | 21524/49172 [00:01<00:01, 23483.32 examples/s]
Dropping Long Sequences (>8192) (num_proc=128):  50%|█████     | 24596/49172 [00:01<00:01, 23335.07 examples/s]
Dropping Long Sequences (>8192) (num_proc=128):  60%|██████    | 29588/49172 [00:01<00:00, 29166.32 examples/s]
Dropping Long Sequences (>8192) (num_proc=128):  69%|██████▉   | 33812/49172 [00:01<00:00, 31889.27 examples/s]
Dropping Long Sequences (>8192) (num_proc=128):  76%|███████▌  | 37268/49172 [00:02<00:00, 32125.51 examples/s]
Dropping Long Sequences (>8192) (num_proc=128):  83%|████████▎ | 40724/49172 [00:02<00:00, 31182.70 examples/s]
Dropping Long Sequences (>8192) (num_proc=128):  93%|█████████▎| 45716/49172 [00:02<00:00, 33653.89 examples/s]
Dropping Long Sequences (>8192) (num_proc=128): 100%|██████████| 49172/49172 [00:02<00:00, 18442.91 examples/s]

Saving the dataset (0/128 shards):   0%|          | 0/49172 [00:00<?, ? examples/s]
Saving the dataset (0/128 shards):   1%|          | 385/49172 [00:02<04:27, 182.62 examples/s]
Saving the dataset (1/128 shards):   1%|          | 385/49172 [00:02<04:27, 182.62 examples/s]
Saving the dataset (2/128 shards):   2%|▏         | 770/49172 [00:02<04:25, 182.62 examples/s]
Saving the dataset (3/128 shards):   2%|▏         | 1155/49172 [00:02<04:22, 182.62 examples/s]
Saving the dataset (4/128 shards):   3%|▎         | 1540/49172 [00:02<04:20, 182.62 examples/s]
Saving the dataset (5/128 shards):   4%|▍         | 1925/49172 [00:02<04:18, 182.62 examples/s]
Saving the dataset (6/128 shards):   5%|▍         | 2310/49172 [00:02<04:16, 182.62 examples/s]
Saving the dataset (7/128 shards):   5%|▌         | 2695/49172 [00:02<04:14, 182.62 examples/s]
Saving the dataset (8/128 shards):   6%|▋         | 3080/49172 [00:02<04:12, 182.62 examples/s]
Saving the dataset (9/128 shards):   7%|▋         | 3465/49172 [00:02<04:10, 182.62 examples/s]
Saving the dataset (10/128 shards):   8%|▊         | 3850/49172 [00:02<04:08, 182.62 examples/s]
Saving the dataset (11/128 shards):   9%|▊         | 4235/49172 [00:02<04:06, 182.62 examples/s]
Saving the dataset (12/128 shards):   9%|▉         | 4620/49172 [00:02<04:03, 182.62 examples/s]
Saving the dataset (13/128 shards):  10%|█         | 5005/49172 [00:02<04:01, 182.62 examples/s]
Saving the dataset (14/128 shards):  11%|█         | 5390/49172 [00:02<03:59, 182.62 examples/s]
Saving the dataset (15/128 shards):  12%|█▏        | 5775/49172 [00:02<03:57, 182.62 examples/s]
Saving the dataset (16/128 shards):  13%|█▎        | 6160/49172 [00:02<03:55, 182.62 examples/s]
Saving the dataset (17/128 shards):  13%|█▎        | 6545/49172 [00:02<03:53, 182.62 examples/s]
Saving the dataset (18/128 shards):  14%|█▍        | 6930/49172 [00:02<03:51, 182.62 examples/s]
Saving the dataset (19/128 shards):  15%|█▍        | 7315/49172 [00:02<03:49, 182.62 examples/s]
Saving the dataset (20/128 shards):  16%|█▌        | 7700/49172 [00:02<03:47, 182.62 examples/s]
Saving the dataset (21/128 shards):  16%|█▋        | 8084/49172 [00:02<03:44, 182.62 examples/s]
Saving the dataset (22/128 shards):  17%|█▋        | 8468/49172 [00:02<03:42, 182.62 examples/s]
Saving the dataset (23/128 shards):  18%|█▊        | 8852/49172 [00:02<03:40, 182.62 examples/s]
Saving the dataset (24/128 shards):  19%|█▉        | 9236/49172 [00:02<03:38, 182.62 examples/s]
Saving the dataset (25/128 shards):  20%|█▉        | 9620/49172 [00:02<03:36, 182.62 examples/s]
Saving the dataset (26/128 shards):  20%|██        | 10004/49172 [00:02<03:34, 182.62 examples/s]
Saving the dataset (27/128 shards):  21%|██        | 10388/49172 [00:02<03:32, 182.62 examples/s]
Saving the dataset (28/128 shards):  22%|██▏       | 10772/49172 [00:02<03:30, 182.62 examples/s]
Saving the dataset (29/128 shards):  23%|██▎       | 11156/49172 [00:02<03:28, 182.62 examples/s]
Saving the dataset (30/128 shards):  23%|██▎       | 11540/49172 [00:02<03:26, 182.62 examples/s]
Saving the dataset (31/128 shards):  24%|██▍       | 11924/49172 [00:02<03:23, 182.62 examples/s]
Saving the dataset (32/128 shards):  25%|██▌       | 12308/49172 [00:02<03:21, 182.62 examples/s]
Saving the dataset (33/128 shards):  26%|██▌       | 12692/49172 [00:02<03:19, 182.62 examples/s]
Saving the dataset (34/128 shards):  27%|██▋       | 13076/49172 [00:02<03:17, 182.62 examples/s]
Saving the dataset (35/128 shards):  27%|██▋       | 13460/49172 [00:02<03:15, 182.62 examples/s]
Saving the dataset (36/128 shards):  28%|██▊       | 13844/49172 [00:02<03:13, 182.62 examples/s]
Saving the dataset (37/128 shards):  29%|██▉       | 14228/49172 [00:02<03:11, 182.62 examples/s]
Saving the dataset (38/128 shards):  30%|██▉       | 14612/49172 [00:02<03:09, 182.62 examples/s]
Saving the dataset (39/128 shards):  30%|███       | 14996/49172 [00:02<03:07, 182.62 examples/s]
Saving the dataset (40/128 shards):  31%|███▏      | 15380/49172 [00:02<03:05, 182.62 examples/s]
Saving the dataset (41/128 shards):  32%|███▏      | 15764/49172 [00:02<03:02, 182.62 examples/s]
Saving the dataset (42/128 shards):  33%|███▎      | 16148/49172 [00:02<03:00, 182.62 examples/s]
Saving the dataset (43/128 shards):  34%|███▎      | 16532/49172 [00:02<02:58, 182.62 examples/s]
Saving the dataset (44/128 shards):  34%|███▍      | 16916/49172 [00:02<02:56, 182.62 examples/s]
Saving the dataset (45/128 shards):  35%|███▌      | 17300/49172 [00:02<02:54, 182.62 examples/s]
Saving the dataset (46/128 shards):  36%|███▌      | 17684/49172 [00:02<02:52, 182.62 examples/s]
Saving the dataset (47/128 shards):  37%|███▋      | 18068/49172 [00:02<02:50, 182.62 examples/s]
Saving the dataset (48/128 shards):  38%|███▊      | 18452/49172 [00:02<02:48, 182.62 examples/s]
Saving the dataset (49/128 shards):  38%|███▊      | 18836/49172 [00:02<02:46, 182.62 examples/s]
Saving the dataset (50/128 shards):  39%|███▉      | 19220/49172 [00:02<02:44, 182.62 examples/s]
Saving the dataset (51/128 shards):  40%|███▉      | 19604/49172 [00:02<02:41, 182.62 examples/s]
Saving the dataset (52/128 shards):  41%|████      | 19988/49172 [00:02<02:39, 182.62 examples/s]
Saving the dataset (53/128 shards):  41%|████▏     | 20372/49172 [00:02<02:37, 182.62 examples/s]
Saving the dataset (54/128 shards):  42%|████▏     | 20756/49172 [00:02<02:35, 182.62 examples/s]
Saving the dataset (55/128 shards):  43%|████▎     | 21140/49172 [00:02<02:33, 182.62 examples/s]
Saving the dataset (56/128 shards):  44%|████▍     | 21524/49172 [00:02<02:31, 182.62 examples/s]
Saving the dataset (57/128 shards):  45%|████▍     | 21908/49172 [00:02<02:29, 182.62 examples/s]
Saving the dataset (58/128 shards):  45%|████▌     | 22292/49172 [00:02<02:27, 182.62 examples/s]
Saving the dataset (59/128 shards):  46%|████▌     | 22676/49172 [00:02<02:25, 182.62 examples/s]
Saving the dataset (60/128 shards):  47%|████▋     | 23060/49172 [00:02<02:22, 182.62 examples/s]
Saving the dataset (61/128 shards):  48%|████▊     | 23444/49172 [00:02<02:20, 182.62 examples/s]
Saving the dataset (62/128 shards):  48%|████▊     | 23828/49172 [00:02<02:18, 182.62 examples/s]
Saving the dataset (63/128 shards):  49%|████▉     | 24212/49172 [00:02<02:16, 182.62 examples/s]
Saving the dataset (64/128 shards):  50%|█████     | 24596/49172 [00:02<02:14, 182.62 examples/s]
Saving the dataset (65/128 shards):  51%|█████     | 24980/49172 [00:02<02:12, 182.62 examples/s]
Saving the dataset (66/128 shards):  52%|█████▏    | 25748/49172 [00:02<02:08, 182.62 examples/s]
Saving the dataset (67/128 shards):  52%|█████▏    | 25748/49172 [00:02<02:08, 182.62 examples/s]
Saving the dataset (68/128 shards):  53%|█████▎    | 26132/49172 [00:02<02:06, 182.62 examples/s]
Saving the dataset (69/128 shards):  54%|█████▍    | 26516/49172 [00:02<02:04, 182.62 examples/s]
Saving the dataset (70/128 shards):  55%|█████▍    | 26900/49172 [00:02<02:01, 182.62 examples/s]
Saving the dataset (71/128 shards):  55%|█████▌    | 27284/49172 [00:02<01:59, 182.62 examples/s]
Saving the dataset (72/128 shards):  56%|█████▋    | 27668/49172 [00:02<01:57, 182.62 examples/s]
Saving the dataset (73/128 shards):  57%|█████▋    | 28052/49172 [00:02<01:55, 182.62 examples/s]
Saving the dataset (74/128 shards):  58%|█████▊    | 28436/49172 [00:02<01:53, 182.62 examples/s]
Saving the dataset (75/128 shards):  59%|█████▊    | 28820/49172 [00:02<01:51, 182.62 examples/s]
Saving the dataset (76/128 shards):  59%|█████▉    | 29204/49172 [00:02<01:49, 182.62 examples/s]
Saving the dataset (77/128 shards):  60%|██████    | 29588/49172 [00:02<01:47, 182.62 examples/s]
Saving the dataset (78/128 shards):  61%|██████    | 29972/49172 [00:02<01:45, 182.62 examples/s]
Saving the dataset (79/128 shards):  62%|██████▏   | 30356/49172 [00:02<01:43, 182.62 examples/s]
Saving the dataset (80/128 shards):  63%|██████▎   | 30740/49172 [00:02<01:40, 182.62 examples/s]
Saving the dataset (81/128 shards):  63%|██████▎   | 31124/49172 [00:02<01:38, 182.62 examples/s]
Saving the dataset (82/128 shards):  64%|██████▍   | 31508/49172 [00:02<01:36, 182.62 examples/s]
Saving the dataset (83/128 shards):  65%|██████▍   | 31892/49172 [00:02<01:34, 182.62 examples/s]
Saving the dataset (84/128 shards):  66%|██████▌   | 32276/49172 [00:02<01:32, 182.62 examples/s]
Saving the dataset (85/128 shards):  66%|██████▋   | 32660/49172 [00:02<01:30, 182.62 examples/s]
Saving the dataset (86/128 shards):  67%|██████▋   | 33044/49172 [00:02<01:28, 182.62 examples/s]
Saving the dataset (87/128 shards):  68%|██████▊   | 33428/49172 [00:02<01:26, 182.62 examples/s]
Saving the dataset (88/128 shards):  69%|██████▉   | 33812/49172 [00:02<01:24, 182.62 examples/s]
Saving the dataset (89/128 shards):  70%|███████   | 34580/49172 [00:02<01:19, 182.62 examples/s]
Saving the dataset (90/128 shards):  70%|███████   | 34580/49172 [00:02<01:19, 182.62 examples/s]
Saving the dataset (91/128 shards):  71%|███████   | 34964/49172 [00:02<01:17, 182.62 examples/s]
Saving the dataset (92/128 shards):  72%|███████▏  | 35348/49172 [00:02<01:15, 182.62 examples/s]
Saving the dataset (93/128 shards):  73%|███████▎  | 35732/49172 [00:02<01:13, 182.62 examples/s]
Saving the dataset (94/128 shards):  73%|███████▎  | 36116/49172 [00:02<01:11, 182.62 examples/s]
Saving the dataset (95/128 shards):  74%|███████▍  | 36500/49172 [00:02<01:09, 182.62 examples/s]
Saving the dataset (96/128 shards):  75%|███████▌  | 36884/49172 [00:02<01:07, 182.62 examples/s]
Saving the dataset (97/128 shards):  76%|███████▌  | 37268/49172 [00:02<01:05, 182.62 examples/s]
Saving the dataset (98/128 shards):  77%|███████▋  | 38036/49172 [00:02<01:00, 182.62 examples/s]
Saving the dataset (99/128 shards):  77%|███████▋  | 38036/49172 [00:02<01:00, 182.62 examples/s]
Saving the dataset (100/128 shards):  78%|███████▊  | 38420/49172 [00:02<00:58, 182.62 examples/s]
Saving the dataset (101/128 shards):  79%|███████▉  | 38804/49172 [00:02<00:56, 182.62 examples/s]
Saving the dataset (102/128 shards):  80%|███████▉  | 39188/49172 [00:02<00:54, 182.62 examples/s]
Saving the dataset (103/128 shards):  80%|████████  | 39572/49172 [00:02<00:52, 182.62 examples/s]
Saving the dataset (104/128 shards):  81%|████████▏ | 39956/49172 [00:02<00:50, 182.62 examples/s]
Saving the dataset (105/128 shards):  82%|████████▏ | 40340/49172 [00:02<00:48, 182.62 examples/s]
Saving the dataset (106/128 shards):  83%|████████▎ | 40724/49172 [00:02<00:46, 182.62 examples/s]
Saving the dataset (107/128 shards):  84%|████████▎ | 41108/49172 [00:02<00:44, 182.62 examples/s]
Saving the dataset (108/128 shards):  84%|████████▍ | 41492/49172 [00:02<00:42, 182.62 examples/s]
Saving the dataset (109/128 shards):  85%|████████▌ | 41876/49172 [00:02<00:39, 182.62 examples/s]
Saving the dataset (110/128 shards):  86%|████████▌ | 42260/49172 [00:02<00:37, 182.62 examples/s]
Saving the dataset (111/128 shards):  87%|████████▋ | 42644/49172 [00:02<00:35, 182.62 examples/s]
Saving the dataset (112/128 shards):  88%|████████▊ | 43028/49172 [00:02<00:33, 182.62 examples/s]
Saving the dataset (113/128 shards):  88%|████████▊ | 43412/49172 [00:02<00:31, 182.62 examples/s]
Saving the dataset (114/128 shards):  89%|████████▉ | 43796/49172 [00:02<00:29, 182.62 examples/s]
Saving the dataset (115/128 shards):  90%|████████▉ | 44180/49172 [00:02<00:27, 182.62 examples/s]
Saving the dataset (116/128 shards):  91%|█████████▏| 44948/49172 [00:02<00:23, 182.62 examples/s]
Saving the dataset (117/128 shards):  91%|█████████▏| 44948/49172 [00:02<00:23, 182.62 examples/s]
Saving the dataset (118/128 shards):  92%|█████████▏| 45332/49172 [00:02<00:21, 182.62 examples/s]
Saving the dataset (119/128 shards):  93%|█████████▎| 45716/49172 [00:02<00:18, 182.62 examples/s]
Saving the dataset (120/128 shards):  94%|█████████▍| 46100/49172 [00:02<00:16, 182.62 examples/s]
Saving the dataset (121/128 shards):  95%|█████████▍| 46484/49172 [00:02<00:14, 182.62 examples/s]
Saving the dataset (122/128 shards):  95%|█████████▌| 46868/49172 [00:02<00:12, 182.62 examples/s]
Saving the dataset (123/128 shards):  96%|█████████▌| 47252/49172 [00:02<00:10, 182.62 examples/s]
Saving the dataset (124/128 shards):  97%|█████████▋| 47636/49172 [00:02<00:08, 182.62 examples/s]
Saving the dataset (125/128 shards):  98%|█████████▊| 48020/49172 [00:02<00:06, 182.62 examples/s]
Saving the dataset (126/128 shards):  98%|█████████▊| 48404/49172 [00:02<00:04, 182.62 examples/s]
Saving the dataset (127/128 shards):  99%|█████████▉| 48788/49172 [00:02<00:02, 182.62 examples/s]
Saving the dataset (128/128 shards): 100%|██████████| 49172/49172 [00:02<00:00, 182.62 examples/s]
Saving the dataset (128/128 shards): 100%|██████████| 49172/49172 [00:02<00:00, 22251.27 examples/s]
[2025-11-06 16:12:34,233] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:404] [PID:837642] total_num_tokens: 9_208_425
[2025-11-06 16:12:34,425] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:422] [PID:837642] `total_supervised_tokens: 6_847_432`
[2025-11-06 16:12:34,425] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:520] [PID:837642] total_num_steps: 769
[2025-11-06 16:12:34,425] [INFO] [axolotl.utils.data.sft._prepare_standard_dataset:121] [PID:837642] Maximum number of steps set at 769
[2025-11-06 16:12:34,441] [DEBUG] [axolotl.train.setup_model_and_tokenizer:70] [PID:837642] Loading tokenizer... meta-llama/Llama-3.2-3B
[2025-11-06 16:12:35,271] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:278] [PID:837642] EOS: 128001 / <|end_of_text|>
[2025-11-06 16:12:35,271] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:279] [PID:837642] BOS: 128000 / <|begin_of_text|>
[2025-11-06 16:12:35,271] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:280] [PID:837642] PAD: 128004 / <|finetune_right_pad_id|>
[2025-11-06 16:12:35,271] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:281] [PID:837642] UNK: None / None
[2025-11-06 16:12:35,271] [INFO] [axolotl.loaders.tokenizer.load_tokenizer:295] [PID:837642] No Chat template selected. Consider adding a chat template for easier inference.
[2025-11-06 16:12:35,271] [DEBUG] [axolotl.train.setup_model_and_tokenizer:79] [PID:837642] Loading model
[2025-11-06 16:12:35,502] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_evaluation_loop:87] [PID:837642] Patched Trainer.evaluation_loop with nanmean loss calculation
[2025-11-06 16:12:35,503] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_maybe_log_save_evaluate:138] [PID:837642] Patched Trainer._maybe_log_save_evaluate with nanmean loss calculation
[2025-11-06 16:12:35,531] [INFO] [axolotl.integrations.liger.plugin.pre_model_load:71] [PID:837642] Applying LIGER to llama with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'swiglu': True}

Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]
Loading checkpoint shards: 100%|██████████| 2/2 [00:00<00:00, 86.01it/s]
[2025-11-06 16:15:46,317] [INFO] [axolotl.loaders.model._configure_embedding_dtypes:345] [PID:837642] Converting modules to torch.bfloat16
[2025-11-06 16:15:59,472] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:837642] Memory usage after model load 0.000GB ()
[2025-11-06 16:16:00,600] [WARNING] [accelerate.utils.other.check_os_kernel:512] [PID:837642] Detected kernel version 4.18.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher.
[2025-11-06 16:16:07,189] [INFO] [axolotl.train.save_initial_configs:412] [PID:837642] Pre-saving tokenizer to ./outputs/qat_out/...
[2025-11-06 16:16:07,288] [INFO] [axolotl.train.save_initial_configs:417] [PID:837642] Pre-saving model config to ./outputs/qat_out/...
[2025-11-06 16:16:07,290] [INFO] [axolotl.train.execute_training:203] [PID:837642] Starting trainer...

  0%|          | 0/769 [00:00<?, ?it/s]
  0%|          | 1/769 [00:10<2:09:18, 10.10s/it]
                                                 
{'loss': 1.1473, 'grad_norm': 4.625, 'learning_rate': 0.0, 'memory/max_active (GiB)': 34.78, 'memory/max_allocated (GiB)': 34.78, 'memory/device_reserved (GiB)': 41.06, 'tokens_per_second_per_gpu': 879.01, 'epoch': 0.0}

  0%|          | 1/769 [00:10<2:09:18, 10.10s/it][2025-11-06 16:16:17,683] [INFO] [axolotl.core.trainers.base._save:671] [PID:837642] Saving model checkpoint to ./outputs/qat_out/checkpoint-1

  0%|          | 2/769 [00:28<3:13:22, 15.13s/it]
                                                 
{'loss': 1.1048, 'grad_norm': 4.34375, 'learning_rate': 2.6315789473684213e-07, 'memory/max_active (GiB)': 46.77, 'memory/max_allocated (GiB)': 46.77, 'memory/device_reserved (GiB)': 55.05, 'tokens_per_second_per_gpu': 1653.05, 'epoch': 0.0}

  0%|          | 2/769 [00:28<3:13:22, 15.13s/it]
  0%|          | 3/769 [00:35<2:23:03, 11.21s/it]
                                                 
{'loss': 1.1442, 'grad_norm': 4.6875, 'learning_rate': 5.263157894736843e-07, 'memory/max_active (GiB)': 49.73, 'memory/max_allocated (GiB)': 49.73, 'memory/device_reserved (GiB)': 76.38, 'tokens_per_second_per_gpu': 1475.08, 'epoch': 0.0}

  0%|          | 3/769 [00:35<2:23:03, 11.21s/it]
  1%|          | 4/769 [00:40<1:55:11,  9.03s/it]
                                                 
{'loss': 1.1473, 'grad_norm': 3.671875, 'learning_rate': 7.894736842105263e-07, 'memory/max_active (GiB)': 46.8, 'memory/max_allocated (GiB)': 46.8, 'memory/device_reserved (GiB)': 76.38, 'tokens_per_second_per_gpu': 2080.31, 'epoch': 0.01}

  1%|          | 4/769 [00:41<1:55:11,  9.03s/it]
  1%|          | 5/769 [00:46<1:39:41,  7.83s/it]
                                                 
{'loss': 1.1704, 'grad_norm': 4.15625, 'learning_rate': 1.0526315789473685e-06, 'memory/max_active (GiB)': 46.77, 'memory/max_allocated (GiB)': 46.77, 'memory/device_reserved (GiB)': 76.38, 'tokens_per_second_per_gpu': 1673.31, 'epoch': 0.01}

  1%|          | 5/769 [00:46<1:39:41,  7.83s/it]
  1%|          | 6/769 [00:51<1:28:32,  6.96s/it]
                                                 
{'loss': 1.1557, 'grad_norm': 4.09375, 'learning_rate': 1.3157894736842106e-06, 'memory/max_active (GiB)': 43.82, 'memory/max_allocated (GiB)': 43.82, 'memory/device_reserved (GiB)': 78.38, 'tokens_per_second_per_gpu': 1894.62, 'epoch': 0.01}

  1%|          | 6/769 [00:51<1:28:32,  6.96s/it]
  1%|          | 7/769 [00:57<1:20:49,  6.36s/it]
                                                 
{'loss': 1.1819, 'grad_norm': 5.21875, 'learning_rate': 1.5789473684210526e-06, 'memory/max_active (GiB)': 43.82, 'memory/max_allocated (GiB)': 43.82, 'memory/device_reserved (GiB)': 77.51, 'tokens_per_second_per_gpu': 1854.54, 'epoch': 0.01}

  1%|          | 7/769 [00:57<1:20:49,  6.36s/it]
  1%|          | 8/769 [01:02<1:18:07,  6.16s/it]
                                                 
{'loss': 1.1807, 'grad_norm': 4.125, 'learning_rate': 1.8421052631578948e-06, 'memory/max_active (GiB)': 46.78, 'memory/max_allocated (GiB)': 46.78, 'memory/device_reserved (GiB)': 78.38, 'tokens_per_second_per_gpu': 1751.39, 'epoch': 0.01}

  1%|          | 8/769 [01:02<1:18:07,  6.16s/it]
  1%|          | 9/769 [01:08<1:16:07,  6.01s/it]
                                                 
{'loss': 1.1324, 'grad_norm': 4.40625, 'learning_rate': 2.105263157894737e-06, 'memory/max_active (GiB)': 46.75, 'memory/max_allocated (GiB)': 46.75, 'memory/device_reserved (GiB)': 77.88, 'tokens_per_second_per_gpu': 1412.84, 'epoch': 0.01}

  1%|          | 9/769 [01:08<1:16:07,  6.01s/it]
  1%|▏         | 10/769 [01:13<1:12:34,  5.74s/it]
                                                  
{'loss': 1.1556, 'grad_norm': 3.78125, 'learning_rate': 2.368421052631579e-06, 'memory/max_active (GiB)': 43.82, 'memory/max_allocated (GiB)': 43.82, 'memory/device_reserved (GiB)': 77.88, 'tokens_per_second_per_gpu': 1849.28, 'epoch': 0.01}

  1%|▏         | 10/769 [01:13<1:12:34,  5.74s/it]
  1%|▏         | 11/769 [01:19<1:12:17,  5.72s/it]
                                                  
{'loss': 1.1855, 'grad_norm': 3.859375, 'learning_rate': 2.631578947368421e-06, 'memory/max_active (GiB)': 46.77, 'memory/max_allocated (GiB)': 46.77, 'memory/device_reserved (GiB)': 77.88, 'tokens_per_second_per_gpu': 1624.94, 'epoch': 0.01}

  1%|▏         | 11/769 [01:19<1:12:17,  5.72s/it]
  2%|▏         | 12/769 [01:24<1:12:03,  5.71s/it]
                                                  
{'loss': 1.1024, 'grad_norm': 4.09375, 'learning_rate': 2.8947368421052634e-06, 'memory/max_active (GiB)': 46.75, 'memory/max_allocated (GiB)': 46.75, 'memory/device_reserved (GiB)': 77.88, 'tokens_per_second_per_gpu': 1463.62, 'epoch': 0.02}

  2%|▏         | 12/769 [01:25<1:12:03,  5.71s/it]
  2%|▏         | 13/769 [01:29<1:08:10,  5.41s/it]
                                                  
{'loss': 1.1591, 'grad_norm': 3.765625, 'learning_rate': 3.157894736842105e-06, 'memory/max_active (GiB)': 41.5, 'memory/max_allocated (GiB)': 41.5, 'memory/device_reserved (GiB)': 78.38, 'tokens_per_second_per_gpu': 1910.89, 'epoch': 0.02}

  2%|▏         | 13/769 [01:29<1:08:10,  5.41s/it]
  2%|▏         | 14/769 [01:35<1:09:08,  5.49s/it]
                                                  
{'loss': 1.0844, 'grad_norm': 3.484375, 'learning_rate': 3.421052631578948e-06, 'memory/max_active (GiB)': 46.76, 'memory/max_allocated (GiB)': 46.76, 'memory/device_reserved (GiB)': 77.88, 'tokens_per_second_per_gpu': 1588.42, 'epoch': 0.02}

  2%|▏         | 14/769 [01:35<1:09:08,  5.49s/it]
  2%|▏         | 15/769 [01:39<1:05:28,  5.21s/it]
                                                  
{'loss': 1.2444, 'grad_norm': 3.984375, 'learning_rate': 3.6842105263157896e-06, 'memory/max_active (GiB)': 41.46, 'memory/max_allocated (GiB)': 41.46, 'memory/device_reserved (GiB)': 77.88, 'tokens_per_second_per_gpu': 1568.51, 'epoch': 0.02}

  2%|▏         | 15/769 [01:39<1:05:28,  5.21s/it]
  2%|▏         | 16/769 [01:45<1:07:09,  5.35s/it]
                                                  
{'loss': 1.1609, 'grad_norm': 3.578125, 'learning_rate': 3.947368421052632e-06, 'memory/max_active (GiB)': 46.74, 'memory/max_allocated (GiB)': 46.74, 'memory/device_reserved (GiB)': 77.88, 'tokens_per_second_per_gpu': 1338.16, 'epoch': 0.02}

  2%|▏         | 16/769 [01:45<1:07:09,  5.35s/it]
  2%|▏         | 17/769 [01:51<1:08:21,  5.45s/it]
                                                  
{'loss': 1.099, 'grad_norm': 3.125, 'learning_rate': 4.210526315789474e-06, 'memory/max_active (GiB)': 46.77, 'memory/max_allocated (GiB)': 46.77, 'memory/device_reserved (GiB)': 77.88, 'tokens_per_second_per_gpu': 1704.75, 'epoch': 0.02}

  2%|▏         | 17/769 [01:51<1:08:21,  5.45s/it]
  2%|▏         | 18/769 [01:57<1:09:09,  5.53s/it]
                                                  
{'loss': 1.0828, 'grad_norm': 3.046875, 'learning_rate': 4.473684210526316e-06, 'memory/max_active (GiB)': 46.77, 'memory/max_allocated (GiB)': 46.77, 'memory/device_reserved (GiB)': 77.88, 'tokens_per_second_per_gpu': 1664.81, 'epoch': 0.02}

  2%|▏         | 18/769 [01:57<1:09:09,  5.53s/it]
  2%|▏         | 19/769 [02:02<1:07:39,  5.41s/it]
                                                  
{'loss': 1.2319, 'grad_norm': 3.4375, 'learning_rate': 4.736842105263158e-06, 'memory/max_active (GiB)': 43.8, 'memory/max_allocated (GiB)': 43.8, 'memory/device_reserved (GiB)': 78.38, 'tokens_per_second_per_gpu': 1602.68, 'epoch': 0.02}

  2%|▏         | 19/769 [02:02<1:07:39,  5.41s/it]
  3%|▎         | 20/769 [02:07<1:08:44,  5.51s/it]
                                                  
{'loss': 1.0638, 'grad_norm': 2.953125, 'learning_rate': 5e-06, 'memory/max_active (GiB)': 46.78, 'memory/max_allocated (GiB)': 46.78, 'memory/device_reserved (GiB)': 78.13, 'tokens_per_second_per_gpu': 1755.41, 'epoch': 0.03}

  3%|▎         | 20/769 [02:07<1:08:44,  5.51s/it]
  3%|▎         | 21/769 [02:13<1:09:22,  5.56s/it]
                                                  
{'loss': 1.2072, 'grad_norm': 2.859375, 'learning_rate': 5.263157894736842e-06, 'memory/max_active (GiB)': 46.78, 'memory/max_allocated (GiB)': 46.78, 'memory/device_reserved (GiB)': 77.38, 'tokens_per_second_per_gpu': 1833.74, 'epoch': 0.03}

  3%|▎         | 21/769 [02:13<1:09:22,  5.56s/it]
  3%|▎         | 22/769 [02:18<1:07:37,  5.43s/it]
                                                  
{'loss': 1.2006, 'grad_norm': 3.109375, 'learning_rate': 5.526315789473685e-06, 'memory/max_active (GiB)': 43.79, 'memory/max_allocated (GiB)': 43.79, 'memory/device_reserved (GiB)': 77.38, 'tokens_per_second_per_gpu': 1567.66, 'epoch': 0.03}

  3%|▎         | 22/769 [02:18<1:07:37,  5.43s/it]
  3%|▎         | 23/769 [02:25<1:12:34,  5.84s/it]
                                                  
{'loss': 1.0081, 'grad_norm': 2.625, 'learning_rate': 5.789473684210527e-06, 'memory/max_active (GiB)': 49.73, 'memory/max_allocated (GiB)': 49.73, 'memory/device_reserved (GiB)': 78.38, 'tokens_per_second_per_gpu': 1412.25, 'epoch': 0.03}

  3%|▎         | 23/769 [02:25<1:12:34,  5.84s/it]
  3%|▎         | 24/769 [02:31<1:12:04,  5.80s/it]
                                                  
{'loss': 1.1437, 'grad_norm': 2.6875, 'learning_rate': 6.0526315789473685e-06, 'memory/max_active (GiB)': 46.78, 'memory/max_allocated (GiB)': 46.78, 'memory/device_reserved (GiB)': 78.26, 'tokens_per_second_per_gpu': 1749.27, 'epoch': 0.03}

  3%|▎         | 24/769 [02:31<1:12:04,  5.80s/it]
  3%|▎         | 25/769 [02:36<1:11:33,  5.77s/it]
                                                  
{'loss': 1.0987, 'grad_norm': 2.765625, 'learning_rate': 6.31578947368421e-06, 'memory/max_active (GiB)': 46.77, 'memory/max_allocated (GiB)': 46.77, 'memory/device_reserved (GiB)': 77.38, 'tokens_per_second_per_gpu': 1627.64, 'epoch': 0.03}

  3%|▎         | 25/769 [02:36<1:11:33,  5.77s/it]
  3%|▎         | 26/769 [02:42<1:09:06,  5.58s/it]
                                                  
{'loss': 1.1046, 'grad_norm': 2.6875, 'learning_rate': 6.578947368421054e-06, 'memory/max_active (GiB)': 43.82, 'memory/max_allocated (GiB)': 43.82, 'memory/device_reserved (GiB)': 77.38, 'tokens_per_second_per_gpu': 1967.95, 'epoch': 0.03}

  3%|▎         | 26/769 [02:42<1:09:06,  5.58s/it]
  4%|▎         | 27/769 [02:47<1:09:27,  5.62s/it]
                                                  
{'loss': 1.1072, 'grad_norm': 2.8125, 'learning_rate': 6.842105263157896e-06, 'memory/max_active (GiB)': 46.76, 'memory/max_allocated (GiB)': 46.76, 'memory/device_reserved (GiB)': 77.38, 'tokens_per_second_per_gpu': 1594.62, 'epoch': 0.04}

  4%|▎         | 27/769 [02:47<1:09:27,  5.62s/it]
  4%|▎         | 28/769 [02:52<1:05:29,  5.30s/it]
                                                  
{'loss': 1.1974, 'grad_norm': 3.03125, 'learning_rate': 7.1052631578947375e-06, 'memory/max_active (GiB)': 41.47, 'memory/max_allocated (GiB)': 41.47, 'memory/device_reserved (GiB)': 77.38, 'tokens_per_second_per_gpu': 1766.49, 'epoch': 0.04}

  4%|▎         | 28/769 [02:52<1:05:29,  5.30s/it]
  4%|▍         | 29/769 [02:58<1:06:52,  5.42s/it]
                                                  
{'loss': 1.2164, 'grad_norm': 2.734375, 'learning_rate': 7.368421052631579e-06, 'memory/max_active (GiB)': 46.77, 'memory/max_allocated (GiB)': 46.77, 'memory/device_reserved (GiB)': 77.38, 'tokens_per_second_per_gpu': 1593.41, 'epoch': 0.04}

  4%|▍         | 29/769 [02:58<1:06:52,  5.42s/it]
  4%|▍         | 30/769 [03:03<1:07:46,  5.50s/it]
                                                  
{'loss': 1.1324, 'grad_norm': 2.65625, 'learning_rate': 7.631578947368423e-06, 'memory/max_active (GiB)': 46.76, 'memory/max_allocated (GiB)': 46.76, 'memory/device_reserved (GiB)': 77.38, 'tokens_per_second_per_gpu': 1607.66, 'epoch': 0.04}

  4%|▍         | 30/769 [03:03<1:07:46,  5.50s/it]
  4%|▍         | 31/769 [03:09<1:08:21,  5.56s/it]
                                                  
{'loss': 1.0693, 'grad_norm': 2.96875, 'learning_rate': 7.894736842105265e-06, 'memory/max_active (GiB)': 46.75, 'memory/max_allocated (GiB)': 46.75, 'memory/device_reserved (GiB)': 77.38, 'tokens_per_second_per_gpu': 1454.28, 'epoch': 0.04}

  4%|▍         | 31/769 [03:09<1:08:21,  5.56s/it]
  4%|▍         | 32/769 [03:14<1:06:43,  5.43s/it]
                                                  
{'loss': 1.0988, 'grad_norm': 2.65625, 'learning_rate': 8.157894736842106e-06, 'memory/max_active (GiB)': 43.82, 'memory/max_allocated (GiB)': 43.82, 'memory/device_reserved (GiB)': 77.38, 'tokens_per_second_per_gpu': 1972.66, 'epoch': 0.04}

  4%|▍         | 32/769 [03:14<1:06:43,  5.43s/it]
  4%|▍         | 33/769 [03:19<1:03:27,  5.17s/it]
                                                  
{'loss': 1.074, 'grad_norm': 2.84375, 'learning_rate': 8.421052631578948e-06, 'memory/max_active (GiB)': 41.48, 'memory/max_allocated (GiB)': 41.48, 'memory/device_reserved (GiB)': 77.13, 'tokens_per_second_per_gpu': 1705.8, 'epoch': 0.04}

  4%|▍         | 33/769 [03:19<1:03:27,  5.17s/it]
  4%|▍         | 34/769 [03:24<1:03:14,  5.16s/it]
                                                  
{'loss': 1.1673, 'grad_norm': 2.671875, 'learning_rate': 8.68421052631579e-06, 'memory/max_active (GiB)': 43.81, 'memory/max_allocated (GiB)': 43.81, 'memory/device_reserved (GiB)': 76.88, 'tokens_per_second_per_gpu': 1761.02, 'epoch': 0.04}

  4%|▍         | 34/769 [03:24<1:03:14,  5.16s/it]
  5%|▍         | 35/769 [03:29<1:02:59,  5.15s/it]
                                                  
{'loss': 1.1655, 'grad_norm': 3.03125, 'learning_rate': 8.947368421052632e-06, 'memory/max_active (GiB)': 43.79, 'memory/max_allocated (GiB)': 43.79, 'memory/device_reserved (GiB)': 76.01, 'tokens_per_second_per_gpu': 1479.42, 'epoch': 0.05}

  5%|▍         | 35/769 [03:29<1:02:59,  5.15s/it]
  5%|▍         | 36/769 [03:35<1:04:53,  5.31s/it]
                                                  
{'loss': 1.1483, 'grad_norm': 2.71875, 'learning_rate': 9.210526315789474e-06, 'memory/max_active (GiB)': 46.75, 'memory/max_allocated (GiB)': 46.75, 'memory/device_reserved (GiB)': 77.38, 'tokens_per_second_per_gpu': 1466.59, 'epoch': 0.05}

  5%|▍         | 36/769 [03:35<1:04:53,  5.31s/it]
  5%|▍         | 37/769 [03:40<1:04:09,  5.26s/it]
                                                  
{'loss': 1.159, 'grad_norm': 2.578125, 'learning_rate': 9.473684210526315e-06, 'memory/max_active (GiB)': 43.81, 'memory/max_allocated (GiB)': 43.81, 'memory/device_reserved (GiB)': 77.38, 'tokens_per_second_per_gpu': 1806.68, 'epoch': 0.05}

  5%|▍         | 37/769 [03:40<1:04:09,  5.26s/it]
  5%|▍         | 38/769 [03:45<1:05:41,  5.39s/it]
                                                  
{'loss': 1.1035, 'grad_norm': 2.625, 'learning_rate': 9.736842105263159e-06, 'memory/max_active (GiB)': 46.77, 'memory/max_allocated (GiB)': 46.77, 'memory/device_reserved (GiB)': 77.38, 'tokens_per_second_per_gpu': 1689.12, 'epoch': 0.05}

  5%|▍         | 38/769 [03:45<1:05:41,  5.39s/it]
  5%|▌         | 39/769 [03:51<1:06:41,  5.48s/it]
                                                  
{'loss': 1.0471, 'grad_norm': 2.609375, 'learning_rate': 1e-05, 'memory/max_active (GiB)': 46.77, 'memory/max_allocated (GiB)': 46.77, 'memory/device_reserved (GiB)': 77.38, 'tokens_per_second_per_gpu': 1579.8, 'epoch': 0.05}

  5%|▌         | 39/769 [03:51<1:06:41,  5.48s/it]
  5%|▌         | 40/769 [03:56<1:05:20,  5.38s/it]
                                                  
{'loss': 1.2091, 'grad_norm': 2.96875, 'learning_rate': 1.0263157894736844e-05, 'memory/max_active (GiB)': 43.8, 'memory/max_allocated (GiB)': 43.8, 'memory/device_reserved (GiB)': 77.38, 'tokens_per_second_per_gpu': 1542.1, 'epoch': 0.05}

  5%|▌         | 40/769 [03:56<1:05:20,  5.38s/it]
  5%|▌         | 41/769 [04:02<1:06:25,  5.47s/it]
                                                  
{'loss': 1.0722, 'grad_norm': 2.59375, 'learning_rate': 1.0526315789473684e-05, 'memory/max_active (GiB)': 46.77, 'memory/max_allocated (GiB)': 46.77, 'memory/device_reserved (GiB)': 77.38, 'tokens_per_second_per_gpu': 1585.53, 'epoch': 0.05}

  5%|▌         | 41/769 [04:02<1:06:25,  5.47s/it]