Training in progress, epoch 1

adapter_config.json

@@ -0,0 +1,38 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "nferruz/ProtGPT2",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": null,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "qalora_group_size": 16,
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "c_attn",
+    "c_proj",
+    "c_fc"
+  ],
+  "target_parameters": [],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ab1a9a92ffff873459d1f10e2eb9bc52bee197b09bdcbc4b89dd4e69d5cb082
+size 94409168

config.json

@@ -0,0 +1,53 @@
+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 0,
+  "dtype": "float16",
+  "embd_pdrop": 0.1,
+  "eos_token_id": 0,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_ctx": 1024,
+  "n_embd": 1280,
+  "n_head": 20,
+  "n_inner": null,
+  "n_layer": 36,
+  "n_positions": 2048,
+  "quantization_config": {
+    "_load_in_4bit": true,
+    "_load_in_8bit": false,
+    "bnb_4bit_compute_dtype": "float16",
+    "bnb_4bit_quant_storage": "bfloat16",
+    "bnb_4bit_quant_type": "nf4",
+    "bnb_4bit_use_double_quant": true,
+    "llm_int8_enable_fp32_cpu_offload": false,
+    "llm_int8_has_fp16_weight": false,
+    "llm_int8_skip_modules": null,
+    "llm_int8_threshold": 6.0,
+    "load_in_4bit": true,
+    "load_in_8bit": false,
+    "quant_method": "bitsandbytes"
+  },
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "task_specific_params": {
+    "text-generation": {
+      "do_sample": true,
+      "max_length": 50
+    }
+  },
+  "transformers_version": "4.57.0",
+  "use_cache": false,
+  "vocab_size": 50257
+}

debug.log

@@ -0,0 +1,640 @@
+[2025-10-10 13:10:41,462] [DEBUG] [axolotl.utils.config.log_gpu_memory_usage:127] [PID:24741] baseline 0.000GB (+0.000GB allocated, +0.002GB reserved)
+[2025-10-10 13:10:41,462] [INFO] [axolotl.cli.config.load_cfg:248] [PID:24741] config:
+{
+  "activation_offloading": false,
+  "adapter": "qlora",
+  "axolotl_config_path": "config.yaml",
+  "base_model": "nferruz/ProtGPT2",
+  "base_model_config": "nferruz/ProtGPT2",
+  "batch_size": 2,
+  "bf16": false,
+  "capabilities": {
+    "bf16": true,
+    "compute_capability": "sm_75",
+    "fp8": false,
+    "n_gpu": 1,
+    "n_node": 1
+  },
+  "context_parallel_size": 1,
+  "dataloader_num_workers": 1,
+  "dataloader_pin_memory": true,
+  "dataloader_prefetch_factor": 256,
+  "dataset_processes": 2,
+  "datasets": [
+    {
+      "ds_type": "json",
+      "message_property_mappings": {
+        "content": "content",
+        "role": "role"
+      },
+      "path": "/content/sequences_tokenized.jsonl",
+      "trust_remote_code": false
+    }
+  ],
+  "ddp": false,
+  "device": "cuda:0",
+  "dion_rank_fraction": 1.0,
+  "dion_rank_multiple_of": 1,
+  "env_capabilities": {
+    "torch_version": "2.8.0"
+  },
+  "eval_batch_size": 2,
+  "eval_causal_lm_metrics": [
+    "sacrebleu",
+    "comet",
+    "ter",
+    "chrf"
+  ],
+  "eval_max_new_tokens": 128,
+  "eval_sample_packing": true,
+  "eval_steps": 0.01,
+  "eval_table_size": 0,
+  "experimental_skip_move_to_device": true,
+  "fp16": true,
+  "gradient_accumulation_steps": 1,
+  "gradient_checkpointing": true,
+  "gradient_checkpointing_kwargs": {
+    "use_reentrant": true
+  },
+  "group_by_length": false,
+  "hub_model_id": "ProtGPT2-Oxido",
+  "include_tkps": true,
+  "is_falcon_derived_model": false,
+  "is_llama_derived_model": false,
+  "is_mistral_derived_model": false,
+  "learning_rate": 0.002,
+  "lisa_layers_attribute": "model.layers",
+  "load_best_model_at_end": false,
+  "load_in_4bit": true,
+  "load_in_8bit": false,
+  "local_rank": 0,
+  "logging_steps": 100,
+  "lora_alpha": 16,
+  "lora_dropout": 0.05,
+  "lora_r": 32,
+  "lora_target_linear": true,
+  "loraplus_lr_embedding": 1e-06,
+  "lr_scheduler": "cosine",
+  "mean_resizing_embeddings": false,
+  "micro_batch_size": 2,
+  "model_config_type": "gpt2",
+  "num_epochs": 3.0,
+  "optimizer": "paged_adamw_32bit",
+  "output_dir": "./qlora-out",
+  "pad_to_sequence_len": true,
+  "pretrain_multipack_attn": true,
+  "profiler_steps_start": 0,
+  "qlora_sharded_model_loading": false,
+  "ray_num_workers": 1,
+  "resources_per_worker": {
+    "GPU": 1
+  },
+  "sample_packing": true,
+  "sample_packing_bin_size": 200,
+  "sample_packing_group_size": 100000,
+  "save_only_model": false,
+  "save_safetensors": true,
+  "save_strategy": "epoch",
+  "sequence_len": 2048,
+  "shuffle_before_merging_datasets": false,
+  "shuffle_merged_datasets": true,
+  "skip_prepare_dataset": false,
+  "special_tokens": {
+    "eos_token": "<|endoftext|>",
+    "pad_token": "<|endoftext|>"
+  },
+  "streaming_multipack_buffer_size": 10000,
+  "strict": false,
+  "tensor_parallel_size": 1,
+  "tf32": false,
+  "tiled_mlp_use_original_mlp": true,
+  "tokenizer_config": "nferruz/ProtGPT2",
+  "tokenizer_save_jinja_files": true,
+  "tokenizer_type": "AutoTokenizer",
+  "torch_dtype": "torch.float16",
+  "train_on_inputs": false,
+  "trl": {
+    "log_completions": false,
+    "mask_truncated_completions": false,
+    "ref_model_mixup_alpha": 0.9,
+    "ref_model_sync_steps": 64,
+    "scale_rewards": true,
+    "sync_ref_model": false,
+    "use_vllm": false,
+    "vllm_server_host": "0.0.0.0",
+    "vllm_server_port": 8000
+  },
+  "type_of_model": "AutoModelForCausalLM",
+  "use_ray": false,
+  "val_set_size": 0.02,
+  "vllm": {
+    "device": "auto",
+    "dtype": "auto",
+    "gpu_memory_utilization": 0.9,
+    "host": "0.0.0.0",
+    "port": 8000
+  },
+  "warmup_steps": 100,
+  "weight_decay": 0.0,
+  "world_size": 1
+}
+[2025-10-10 13:10:42,465] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:278] [PID:24741] EOS: 0 / <|endoftext|>
+[2025-10-10 13:10:42,465] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:279] [PID:24741] BOS: 0 / <|endoftext|>
+[2025-10-10 13:10:42,465] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:280] [PID:24741] PAD: 0 / <|endoftext|>
+[2025-10-10 13:10:42,465] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:281] [PID:24741] UNK: 0 / <|endoftext|>
+[2025-10-10 13:10:42,465] [INFO] [axolotl.loaders.tokenizer.load_tokenizer:295] [PID:24741] No Chat template selected. Consider adding a chat template for easier inference.
+[2025-10-10 13:10:42,466] [INFO] [axolotl.utils.data.shared.load_preprocessed_dataset:476] [PID:24741] Unable to find prepared dataset in last_run_prepared/120d8e2ed44f3c537dc9a20773f86561
+[2025-10-10 13:10:42,466] [INFO] [axolotl.utils.data.sft._load_raw_datasets:320] [PID:24741] Loading raw datasets...
+[2025-10-10 13:10:42,466] [WARNING] [axolotl.utils.data.sft._load_raw_datasets:322] [PID:24741] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`.
+[2025-10-10 13:10:42,802] [INFO] [axolotl.utils.data.wrappers.get_dataset_wrapper:87] [PID:24741] Loading dataset: /content/sequences_tokenized.jsonl with base_type: None and prompt_style: None
+[2025-10-10 13:10:42,820] [INFO] [axolotl.utils.data.utils.handle_long_seq_in_dataset:218] [PID:24741] min_input_len: 6
+[2025-10-10 13:10:42,821] [INFO] [axolotl.utils.data.utils.handle_long_seq_in_dataset:220] [PID:24741] max_input_len: 512
+
+
+
+
+[2025-10-10 13:10:45,620] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:406] [PID:24741] total_num_tokens: 16_570
+[2025-10-10 13:10:45,622] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:424] [PID:24741] `total_supervised_tokens: 16_570`
+[2025-10-10 13:10:48,083] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 0.9218025207519531
+[2025-10-10 13:10:49,020] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 0.9363107681274414
+[2025-10-10 13:10:49,934] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 0.9140019416809082
+[2025-10-10 13:10:50,862] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 0.9271283149719238
+[2025-10-10 13:10:50,882] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+[2025-10-10 13:10:50,882] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:483] [PID:24741] data_loader_len: 4
+[2025-10-10 13:10:50,883] [INFO] [axolotl.utils.trainer.calc_sample_packing_eff_est:499] [PID:24741] sample_packing_eff_est across ranks: [0.8989800347222222]
+[2025-10-10 13:10:50,883] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:511] [PID:24741] sample_packing_eff_est: None
+[2025-10-10 13:10:50,883] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:522] [PID:24741] total_num_steps: 12
+[2025-10-10 13:10:50,893] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:406] [PID:24741] total_num_tokens: 746_874
+[2025-10-10 13:10:50,932] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:424] [PID:24741] `total_supervised_tokens: 746_874`
+[2025-10-10 13:10:52,871] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 0.9358129501342773
+[2025-10-10 13:10:53,781] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 0.9106135368347168
+[2025-10-10 13:10:55,014] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2319858074188232
+[2025-10-10 13:10:56,287] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2722358703613281
+[2025-10-10 13:10:56,287] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [183]
+[2025-10-10 13:10:56,287] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:483] [PID:24741] data_loader_len: 183
+[2025-10-10 13:10:56,287] [INFO] [axolotl.utils.trainer.calc_sample_packing_eff_est:499] [PID:24741] sample_packing_eff_est across ranks: [0.9936909272820164]
+[2025-10-10 13:10:56,287] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:511] [PID:24741] sample_packing_eff_est: 1.0
+[2025-10-10 13:10:56,287] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:522] [PID:24741] total_num_steps: 549
+[2025-10-10 13:10:56,287] [INFO] [axolotl.utils.data.sft._prepare_standard_dataset:121] [PID:24741] Maximum number of steps set at 549
+[2025-10-10 13:10:56,297] [DEBUG] [axolotl.train.setup_model_and_tokenizer:65] [PID:24741] Loading tokenizer... nferruz/ProtGPT2
+[2025-10-10 13:10:57,214] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:278] [PID:24741] EOS: 0 / <|endoftext|>
+[2025-10-10 13:10:57,214] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:279] [PID:24741] BOS: 0 / <|endoftext|>
+[2025-10-10 13:10:57,214] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:280] [PID:24741] PAD: 0 / <|endoftext|>
+[2025-10-10 13:10:57,214] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:281] [PID:24741] UNK: 0 / <|endoftext|>
+[2025-10-10 13:10:57,214] [INFO] [axolotl.loaders.tokenizer.load_tokenizer:295] [PID:24741] No Chat template selected. Consider adding a chat template for easier inference.
+[2025-10-10 13:10:57,215] [DEBUG] [axolotl.train.setup_model_and_tokenizer:74] [PID:24741] Loading model
+[2025-10-10 13:10:57,333] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_evaluation_loop:87] [PID:24741] Patched Trainer.evaluation_loop with nanmean loss calculation
+[2025-10-10 13:10:57,334] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_maybe_log_save_evaluate:138] [PID:24741] Patched Trainer._maybe_log_save_evaluate with nanmean loss calculation
+[2025-10-10 13:10:57,335] [INFO] [axolotl.loaders.patch_manager._apply_multipack_patches:301] [PID:24741] Applying multipack dataloader patch for sample packing...
+[2025-10-10 13:11:27,461] [WARNING] [axolotl.loaders.model._adjust_model_config:273] [PID:24741] increasing model.config.max_position_embeddings from 1024 to 2048
+[2025-10-10 13:11:27,467] [INFO] [axolotl.loaders.model._prepare_model_for_quantization:863] [PID:24741] converting PEFT model w/ prepare_model_for_kbit_training
+[2025-10-10 13:11:27,479] [INFO] [axolotl.loaders.model._configure_embedding_dtypes:345] [PID:24741] Converting modules to torch.float16
+[2025-10-10 13:11:27,481] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:24741] Memory usage after model load 0.849GB (+0.849GB allocated, +0.918GB reserved)
+[2025-10-10 13:11:27,482] [INFO] [axolotl.loaders.adapter.load_lora:80] [PID:24741] found linear modules: ['c_attn', 'c_fc', 'c_proj']
+trainable params: 23,592,960 || all params: 797,623,040 || trainable%: 2.9579
+[2025-10-10 13:11:27,888] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:24741] after adapters 0.618GB (+0.618GB allocated, +1.012GB reserved)
+[2025-10-10 13:11:39,738] [INFO] [axolotl.train.save_initial_configs:398] [PID:24741] Pre-saving adapter config to ./qlora-out...
+[2025-10-10 13:11:39,738] [INFO] [axolotl.train.save_initial_configs:402] [PID:24741] Pre-saving tokenizer to ./qlora-out...
+[2025-10-10 13:11:39,828] [INFO] [axolotl.train.save_initial_configs:407] [PID:24741] Pre-saving model config to ./qlora-out...
+[2025-10-10 13:11:39,836] [INFO] [axolotl.train.execute_training:196] [PID:24741] Starting trainer...
+[2025-10-10 13:11:45,415] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.8461699485778809
+[2025-10-10 13:11:47,771] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 2.3554582595825195
+[2025-10-10 13:11:49,329] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.5573019981384277
+[2025-10-10 13:11:51,006] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.6764421463012695
+[2025-10-10 13:11:51,006] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [183]
+
  0% 0/549 [00:00<?, ?it/s][2025-10-10 13:11:51,161] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 13:11:54,146] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2840440273284912
+[2025-10-10 13:11:55,365] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.218794584274292
+[2025-10-10 13:11:56,599] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2343621253967285
+[2025-10-10 13:11:58,099] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.4991233348846436
+[2025-10-10 13:11:58,099] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:00<00:00,  3.33it/s]
+
 75% 3/4 [00:01<00:00,  1.68it/s]
+
                           
+
                                 
+
  0% 0/549 [00:14<?, ?it/s]
+
+
                                 
  0% 1/549 [00:21<3:16:49, 21.55s/it]
  0% 2/549 [00:25<1:40:35, 11.03s/it]
  1% 3/549 [00:28<1:09:56,  7.69s/it]
  1% 4/549 [00:32<55:34,  6.12s/it]  
  1% 5/549 [00:36<47:37,  5.25s/it]
  1% 6/549 [00:40<42:50,  4.73s/it][2025-10-10 13:12:31,245] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 13:12:33,921] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.245626449584961
+[2025-10-10 13:12:35,290] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.3684089183807373
+[2025-10-10 13:12:37,029] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.738837718963623
+[2025-10-10 13:12:38,274] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2442030906677246
+[2025-10-10 13:12:38,274] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.90it/s]
+
 75% 3/4 [00:02<00:00,  1.36it/s]
+
                                   
+
                                 
+
  1% 6/549 [00:52<42:50,  4.73s/it]
+
+
                                 
  1% 7/549 [00:56<1:17:17,  8.56s/it]
  1% 8/549 [01:00<1:03:22,  7.03s/it]
  2% 9/549 [01:04<54:04,  6.01s/it]  
  2% 10/549 [01:07<47:47,  5.32s/it]
  2% 11/549 [01:11<43:30,  4.85s/it]
  2% 12/549 [01:15<40:33,  4.53s/it][2025-10-10 13:13:06,562] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 13:13:09,050] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2255823612213135
+[2025-10-10 13:13:10,291] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.240588665008545
+[2025-10-10 13:13:11,581] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2900962829589844
+[2025-10-10 13:13:13,335] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.753103256225586
+[2025-10-10 13:13:13,335] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.86it/s]
+
 75% 3/4 [00:02<00:00,  1.34it/s]
+
                                    
+
                                 
+
  2% 12/549 [01:28<40:33,  4.53s/it]
+
+
                                 
  2% 13/549 [01:31<1:12:45,  8.14s/it]
  3% 14/549 [01:35<1:00:59,  6.84s/it]
  3% 15/549 [01:39<52:49,  5.93s/it]  
  3% 16/549 [01:43<47:08,  5.31s/it]
  3% 17/549 [01:47<43:11,  4.87s/it]
  3% 18/549 [01:51<40:28,  4.57s/it][2025-10-10 13:13:42,268] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 13:13:44,777] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.233976125717163
+[2025-10-10 13:13:45,991] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2140934467315674
+[2025-10-10 13:13:47,230] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2387192249298096
+[2025-10-10 13:13:48,517] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2868869304656982
+[2025-10-10 13:13:48,517] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.83it/s]
+
 75% 3/4 [00:02<00:00,  1.32it/s]
+
                                    
+
                                 
+
  3% 18/549 [02:03<40:28,  4.57s/it]
+
+
                                 
  3% 19/549 [02:07<1:12:15,  8.18s/it]
  4% 20/549 [02:11<1:00:54,  6.91s/it]
  4% 21/549 [02:15<52:54,  6.01s/it]  
  4% 22/549 [02:19<47:19,  5.39s/it]
  4% 23/549 [02:23<43:26,  4.96s/it]
  4% 24/549 [02:27<40:38,  4.64s/it][2025-10-10 13:14:18,514] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 13:14:20,985] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2237637042999268
+[2025-10-10 13:14:22,224] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2387983798980713
+[2025-10-10 13:14:23,447] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2227163314819336
+[2025-10-10 13:14:24,682] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2346465587615967
+[2025-10-10 13:14:24,682] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.85it/s]
+
 75% 3/4 [00:02<00:00,  1.33it/s]
+
                                    
+
                                 
+
  4% 24/549 [02:39<40:38,  4.64s/it]
+
+
                                 
  5% 25/549 [02:43<1:10:34,  8.08s/it]
  5% 26/549 [02:47<59:22,  6.81s/it]  
  5% 27/549 [02:51<51:30,  5.92s/it]
  5% 28/549 [02:54<45:59,  5.30s/it]
  5% 29/549 [02:58<42:11,  4.87s/it]
  5% 30/549 [03:02<39:29,  4.57s/it][2025-10-10 13:14:53,875] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 13:14:56,364] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.229734182357788
+[2025-10-10 13:14:57,586] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2214250564575195
+[2025-10-10 13:14:58,806] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.219433307647705
+[2025-10-10 13:15:00,136] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.329803705215454
+[2025-10-10 13:15:00,136] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.85it/s]
+
 75% 3/4 [00:02<00:00,  1.32it/s]
+
                                    
+
                                 
+
  5% 30/549 [03:14<39:29,  4.57s/it]
+
+
                                 
  6% 31/549 [03:18<1:08:40,  7.95s/it]
  6% 32/549 [03:22<57:59,  6.73s/it]  
  6% 33/549 [03:26<50:32,  5.88s/it]
  6% 34/549 [03:30<45:20,  5.28s/it]
  6% 35/549 [03:34<41:45,  4.87s/it]
  7% 36/549 [03:38<39:13,  4.59s/it][2025-10-10 13:15:29,235] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 13:15:31,722] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.212110996246338
+[2025-10-10 13:15:32,954] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.231447458267212
+[2025-10-10 13:15:34,205] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2508065700531006
+[2025-10-10 13:15:35,438] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2330925464630127
+[2025-10-10 13:15:35,439] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.84it/s]
+
 75% 3/4 [00:02<00:00,  1.32it/s]
+
                                    
+
                                 
+
  7% 36/549 [03:50<39:13,  4.59s/it]
+
+
                                 
  7% 37/549 [03:53<1:08:00,  7.97s/it]
  7% 38/549 [03:57<57:23,  6.74s/it]  
  7% 39/549 [04:01<50:02,  5.89s/it]
  7% 40/549 [04:05<44:51,  5.29s/it]
  7% 41/549 [04:09<41:10,  4.86s/it]
  8% 42/549 [04:13<38:40,  4.58s/it][2025-10-10 13:16:04,533] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 13:16:07,240] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2179176807403564
+[2025-10-10 13:16:08,479] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.239612340927124
+[2025-10-10 13:16:09,694] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2147700786590576
+[2025-10-10 13:16:10,944] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2497029304504395
+[2025-10-10 13:16:10,945] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.85it/s]
+
 75% 3/4 [00:02<00:00,  1.33it/s]
+
                                    
+
                                 
+
  8% 42/549 [04:25<38:40,  4.58s/it]
+
+
                                 
  8% 43/549 [04:29<1:07:37,  8.02s/it]
  8% 44/549 [04:33<56:57,  6.77s/it]  
  8% 45/549 [04:37<49:35,  5.90s/it]
  8% 46/549 [04:41<44:25,  5.30s/it]
  9% 47/549 [04:44<40:49,  4.88s/it]
  9% 48/549 [04:48<38:22,  4.60s/it][2025-10-10 13:16:40,037] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 13:16:43,053] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2188963890075684
+[2025-10-10 13:16:44,293] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2395522594451904
+[2025-10-10 13:16:45,513] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.21950364112854
+[2025-10-10 13:16:46,763] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2497367858886719
+[2025-10-10 13:16:46,763] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.84it/s]
+
 75% 3/4 [00:02<00:00,  1.32it/s]
+
                                    
+
                                 
+
  9% 48/549 [05:01<38:22,  4.60s/it]
+
+
                                 
  9% 49/549 [05:05<1:07:36,  8.11s/it]
  9% 50/549 [05:09<56:51,  6.84s/it]  
  9% 51/549 [05:12<49:28,  5.96s/it]
  9% 52/549 [05:16<44:15,  5.34s/it]
 10% 53/549 [05:20<40:31,  4.90s/it]
 10% 54/549 [05:24<36:22,  4.41s/it][2025-10-10 13:17:15,168] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 13:17:18,275] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.8323063850402832
+[2025-10-10 13:17:19,609] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.3336091041564941
+[2025-10-10 13:17:20,827] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2181472778320312
+[2025-10-10 13:17:22,034] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2064990997314453
+[2025-10-10 13:17:22,034] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.85it/s]
+
 75% 3/4 [00:02<00:00,  1.33it/s]
+
                                    
+
                                 
+
 10% 54/549 [05:36<36:22,  4.41s/it]
+
+
                                 
 10% 55/549 [05:40<1:06:03,  8.02s/it]
 10% 56/549 [05:44<55:34,  6.76s/it]  
 10% 57/549 [05:48<48:18,  5.89s/it]
 11% 58/549 [05:52<43:15,  5.29s/it]
 11% 59/549 [05:55<39:46,  4.87s/it]
 11% 60/549 [05:59<37:18,  4.58s/it][2025-10-10 13:17:50,975] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 13:17:53,534] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2939870357513428
+[2025-10-10 13:17:55,286] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.7515311241149902
+[2025-10-10 13:17:56,595] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.3087666034698486
+[2025-10-10 13:17:58,392] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.7966506481170654
+[2025-10-10 13:17:58,392] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.85it/s]
+
 75% 3/4 [00:02<00:00,  1.33it/s]
+
                                    
+
                                 
+
 11% 60/549 [06:13<37:18,  4.58s/it]
+
+
                                 
 11% 61/549 [06:17<1:09:20,  8.52s/it]
 11% 62/549 [06:21<57:52,  7.13s/it]  
 11% 63/549 [06:25<49:53,  6.16s/it]
 12% 64/549 [06:29<44:24,  5.49s/it]
 12% 65/549 [06:33<40:32,  5.03s/it]
 12% 66/549 [06:37<37:50,  4.70s/it][2025-10-10 13:18:28,296] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 13:18:33,461] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 2.976652145385742
+[2025-10-10 13:18:35,243] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.7817871570587158
+[2025-10-10 13:18:36,478] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2347698211669922
+[2025-10-10 13:18:37,729] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2507171630859375
+[2025-10-10 13:18:37,729] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.80it/s]
+
 75% 3/4 [00:02<00:00,  1.31it/s]
+
                                    
+
                                 
+
 12% 66/549 [06:53<37:50,  4.70s/it]
+
+
                                 
 12% 67/549 [06:57<1:14:42,  9.30s/it]
 12% 68/549 [07:01<1:01:33,  7.68s/it]
 13% 69/549 [07:04<52:22,  6.55s/it]  
 13% 70/549 [07:08<45:55,  5.75s/it]
 13% 71/549 [07:12<41:24,  5.20s/it]
 13% 72/549 [07:16<38:15,  4.81s/it][2025-10-10 13:19:07,848] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 13:19:10,905] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.743417501449585
+[2025-10-10 13:19:12,274] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.3681507110595703
+[2025-10-10 13:19:13,507] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.232816457748413
+[2025-10-10 13:19:14,734] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2264103889465332
+[2025-10-10 13:19:14,734] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.84it/s]
+
 75% 3/4 [00:02<00:00,  1.33it/s]
+
                                    
+
                                 
+
 13% 72/549 [07:29<38:15,  4.81s/it]
+
+
                                 
 13% 73/549 [07:33<1:05:59,  8.32s/it]
 13% 74/549 [07:36<53:50,  6.80s/it]  
 14% 75/549 [07:40<46:47,  5.92s/it]
 14% 76/549 [07:44<41:57,  5.32s/it]
 14% 77/549 [07:48<38:32,  4.90s/it]
 14% 78/549 [07:52<36:06,  4.60s/it][2025-10-10 13:19:43,212] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 13:19:45,690] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2276828289031982
+[2025-10-10 13:19:47,405] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.7151846885681152
+[2025-10-10 13:19:48,875] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.4696624279022217
+[2025-10-10 13:19:50,118] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.24239182472229
+[2025-10-10 13:19:50,118] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.85it/s]
+
 75% 3/4 [00:02<00:00,  1.33it/s]
+
                                    
+
                                 
+
 14% 78/549 [08:04<36:06,  4.60s/it]
+
+
                                 
 14% 79/549 [08:08<1:03:59,  8.17s/it]
 15% 80/549 [08:12<53:44,  6.88s/it]  
 15% 81/549 [08:16<46:34,  5.97s/it]
 15% 82/549 [08:20<41:37,  5.35s/it]
 15% 83/549 [08:24<38:10,  4.91s/it]
 15% 84/549 [08:27<35:43,  4.61s/it][2025-10-10 13:20:19,124] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 13:20:21,612] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2227427959442139
+[2025-10-10 13:20:22,917] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.3052830696105957
+[2025-10-10 13:20:24,696] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.7787699699401855
+[2025-10-10 13:20:25,947] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2503879070281982
+[2025-10-10 13:20:25,947] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.85it/s]
+
 75% 3/4 [00:02<00:00,  1.33it/s]
+
                                    
+
                                 
+
 15% 84/549 [08:40<35:43,  4.61s/it]
+
+
                                 
 15% 85/549 [08:44<1:03:06,  8.16s/it]
 16% 86/549 [08:48<53:01,  6.87s/it]  
 16% 87/549 [08:52<45:57,  5.97s/it]
 16% 88/549 [08:56<41:08,  5.35s/it]
 16% 89/549 [08:59<37:43,  4.92s/it]
 16% 90/549 [09:03<35:17,  4.61s/it][2025-10-10 13:20:55,024] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 13:20:57,527] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2476911544799805
+[2025-10-10 13:20:58,765] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2383363246917725
+[2025-10-10 13:21:00,176] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.4102411270141602
+[2025-10-10 13:21:01,923] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.7466182708740234
+[2025-10-10 13:21:01,923] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.85it/s]
+
 75% 3/4 [00:02<00:00,  1.33it/s]
+
                                    
+
                                 
+
 16% 90/549 [09:16<35:17,  4.61s/it]
+
+
                                 
 17% 91/549 [09:20<1:02:22,  8.17s/it]
 17% 92/549 [09:24<52:22,  6.88s/it]  
 17% 93/549 [09:28<45:23,  5.97s/it]
 17% 94/549 [09:31<40:32,  5.35s/it]
 17% 95/549 [09:35<37:09,  4.91s/it]
 17% 96/549 [09:39<34:46,  4.61s/it][2025-10-10 13:21:30,890] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 13:21:33,421] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2245755195617676
+[2025-10-10 13:21:34,670] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2479896545410156
+[2025-10-10 13:21:35,919] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2494275569915771
+[2025-10-10 13:21:37,555] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.6355178356170654
+[2025-10-10 13:21:37,555] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.85it/s]
+
 75% 3/4 [00:02<00:00,  1.33it/s]
+
                                    
+
                                 
+
 17% 96/549 [09:52<34:46,  4.61s/it]
+
+
                                 
 18% 97/549 [09:56<1:02:02,  8.24s/it]
 18% 98/549 [10:00<52:01,  6.92s/it]  
 18% 99/549 [10:04<45:02,  6.00s/it]
 18% 100/549 [10:08<40:10,  5.37s/it]
                                     
+
 18% 100/549 [10:08<40:10,  5.37s/it]
 18% 101/549 [10:12<36:58,  4.95s/it]
 19% 102/549 [10:15<34:32,  4.64s/it][2025-10-10 13:22:07,077] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 13:22:09,610] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.245088815689087
+[2025-10-10 13:22:10,855] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2448465824127197
+[2025-10-10 13:22:12,106] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2508001327514648
+[2025-10-10 13:22:13,381] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2749567031860352
+[2025-10-10 13:22:13,382] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.85it/s]
+
 75% 3/4 [00:02<00:00,  1.33it/s]
+
                                     
+
                                 
+
 19% 102/549 [10:28<34:32,  4.64s/it]
+
+
                                 
 19% 103/549 [10:32<1:00:52,  8.19s/it]
 19% 104/549 [10:36<51:10,  6.90s/it]  
 19% 105/549 [10:40<44:23,  6.00s/it]
 19% 106/549 [10:44<39:36,  5.36s/it]
 19% 107/549 [10:48<36:21,  4.94s/it]
 20% 108/549 [10:51<34:00,  4.63s/it][2025-10-10 13:22:43,069] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 13:22:45,569] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.246150016784668
+[2025-10-10 13:22:46,822] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2529652118682861
+[2025-10-10 13:22:48,105] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2827684879302979
+[2025-10-10 13:22:49,347] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2417457103729248
+[2025-10-10 13:22:49,347] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.85it/s]
+
 75% 3/4 [00:02<00:00,  1.33it/s]
+
                                     
+
                                 
+
 20% 108/549 [11:04<34:00,  4.63s/it]
+
+
                                 
 20% 109/549 [11:08<59:10,  8.07s/it]
 20% 110/549 [11:11<49:49,  6.81s/it]
 20% 111/549 [11:15<43:16,  5.93s/it]
 20% 112/549 [11:19<38:43,  5.32s/it]
 21% 113/549 [11:23<35:29,  4.89s/it]
 21% 114/549 [11:27<33:16,  4.59s/it][2025-10-10 13:23:18,582] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 13:23:21,111] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2601120471954346
+[2025-10-10 13:23:22,364] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2526865005493164
+[2025-10-10 13:23:23,588] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.223961591720581
+[2025-10-10 13:23:24,824] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2359259128570557
+[2025-10-10 13:23:24,824] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.85it/s]
+
 75% 3/4 [00:02<00:00,  1.32it/s]
+
                                     
+
                                 
+
 21% 114/549 [11:39<33:16,  4.59s/it]
+
+
                                 
 21% 115/549 [11:43<57:40,  7.97s/it]
 21% 116/549 [11:47<48:37,  6.74s/it]
 21% 117/549 [11:51<42:19,  5.88s/it]
 21% 118/549 [11:54<37:57,  5.28s/it]
 22% 119/549 [11:58<34:52,  4.87s/it]
 22% 120/549 [12:02<32:40,  4.57s/it][2025-10-10 13:23:53,849] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 13:23:56,384] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2743051052093506
+[2025-10-10 13:23:57,627] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2425775527954102
+[2025-10-10 13:23:58,872] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2450978755950928
+[2025-10-10 13:24:00,110] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2371137142181396
+[2025-10-10 13:24:00,110] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.85it/s]
+
 75% 3/4 [00:02<00:00,  1.33it/s]
+
                                     
+
                                 
+
 22% 120/549 [12:14<32:40,  4.57s/it]
+
+
                                 
 22% 121/549 [12:18<56:50,  7.97s/it]
 22% 122/549 [12:22<47:53,  6.73s/it]
 22% 123/549 [12:26<41:44,  5.88s/it]
 23% 124/549 [12:30<37:24,  5.28s/it]
 23% 125/549 [12:34<34:22,  4.86s/it]
 23% 126/549 [12:38<32:18,  4.58s/it][2025-10-10 13:24:29,185] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 13:24:32,019] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2589247226715088
+[2025-10-10 13:24:33,287] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2684495449066162
+[2025-10-10 13:24:34,521] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2337257862091064
+[2025-10-10 13:24:35,751] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2298574447631836
+[2025-10-10 13:24:35,752] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.85it/s]
+
 75% 3/4 [00:02<00:00,  1.33it/s]
+
                                     
+
                                 
+
 23% 126/549 [12:50<32:18,  4.58s/it]
+
+
                                 
 23% 127/549 [12:54<56:47,  8.08s/it]
 23% 128/549 [12:58<47:47,  6.81s/it]
 23% 129/549 [13:02<41:35,  5.94s/it]
 24% 130/549 [13:05<37:15,  5.33s/it]
 24% 131/549 [13:09<34:11,  4.91s/it]
 24% 132/549 [13:13<32:04,  4.61s/it][2025-10-10 13:25:04,943] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 13:25:08,050] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2962446212768555
+[2025-10-10 13:25:09,299] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2489659786224365
+[2025-10-10 13:25:10,529] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2295067310333252
+[2025-10-10 13:25:11,775] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2457163333892822
+[2025-10-10 13:25:11,775] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.85it/s]
+
 75% 3/4 [00:02<00:00,  1.33it/s]
+
                                     
+
                                 
+
 24% 132/549 [13:26<32:04,  4.61s/it]
+
+
                                 
 24% 133/549 [13:30<56:33,  8.16s/it]
 24% 134/549 [13:34<47:29,  6.87s/it]
 25% 135/549 [13:37<41:08,  5.96s/it]
 25% 136/549 [13:41<36:43,  5.34s/it]
 25% 137/549 [13:45<33:38,  4.90s/it]
 25% 138/549 [13:49<31:27,  4.59s/it][2025-10-10 13:25:40,704] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 13:25:43,878] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.726452112197876
+[2025-10-10 13:25:45,124] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.245727777481079
+[2025-10-10 13:25:46,372] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2481646537780762
+[2025-10-10 13:25:47,600] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2278366088867188
+[2025-10-10 13:25:47,601] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.85it/s]
+
 75% 3/4 [00:02<00:00,  1.33it/s]
+
                                     
+
                                 
+
 25% 138/549 [14:02<31:27,  4.59s/it]
+
+
                                 
 25% 139/549 [14:06<55:44,  8.16s/it]
 26% 140/549 [14:09<46:48,  6.87s/it]
 26% 141/549 [14:13<40:35,  5.97s/it]
 26% 142/549 [14:17<36:14,  5.34s/it]
 26% 143/549 [14:21<33:13,  4.91s/it]
 26% 144/549 [14:25<31:06,  4.61s/it][2025-10-10 13:26:16,597] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 13:26:19,460] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.5904531478881836
+[2025-10-10 13:26:21,049] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.5887093544006348
+[2025-10-10 13:26:22,299] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2505967617034912
+[2025-10-10 13:26:23,529] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.229478120803833
+[2025-10-10 13:26:23,529] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.85it/s]
+
 75% 3/4 [00:02<00:00,  1.33it/s]
+
                                     
+
                                 
+
 26% 144/549 [14:38<31:06,  4.61s/it]
+
+
                                 
 26% 145/549 [14:41<55:08,  8.19s/it]
 27% 146/549 [14:45<46:16,  6.89s/it]
 27% 147/549 [14:49<40:05,  5.98s/it]
 27% 148/549 [14:53<35:50,  5.36s/it]
 27% 149/549 [14:57<32:50,  4.93s/it]
 27% 150/549 [15:01<30:40,  4.61s/it][2025-10-10 13:26:52,575] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 13:26:55,133] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.290666103363037
+[2025-10-10 13:26:56,896] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.7622182369232178
+[2025-10-10 13:26:58,238] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.3421812057495117
+[2025-10-10 13:26:59,467] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2286624908447266
+[2025-10-10 13:26:59,467] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.85it/s]
+
 75% 3/4 [00:02<00:00,  1.33it/s]
+
                                     
+
                                 
+
 27% 150/549 [15:14<30:40,  4.61s/it]
+
+
                                 
 28% 151/549 [15:17<54:17,  8.18s/it]
 28% 152/549 [15:21<45:33,  6.89s/it]
 28% 153/549 [15:25<39:27,  5.98s/it]
 28% 154/549 [15:29<35:14,  5.35s/it]
 28% 155/549 [15:33<32:17,  4.92s/it]
 28% 156/549 [15:36<28:59,  4.43s/it][2025-10-10 13:27:27,880] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 13:27:30,445] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2810289859771729
+[2025-10-10 13:27:31,672] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2266299724578857
+[2025-10-10 13:27:33,392] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.7197673320770264
+[2025-10-10 13:27:34,889] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.4970765113830566
+[2025-10-10 13:27:34,889] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.85it/s]
+
 75% 3/4 [00:02<00:00,  1.33it/s]
+
                                     
+
                                 
+
 28% 156/549 [15:49<28:59,  4.43s/it]
+
+
                                 
 29% 157/549 [15:53<52:50,  8.09s/it]
 29% 158/549 [15:57<44:26,  6.82s/it]
 29% 159/549 [16:01<38:37,  5.94s/it]
 29% 160/549 [16:04<34:31,  5.33s/it]
 29% 161/549 [16:08<31:41,  4.90s/it]
 30% 162/549 [16:12<29:40,  4.60s/it][2025-10-10 13:28:03,961] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 13:28:06,582] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2377257347106934
+[2025-10-10 13:28:07,822] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2394213676452637
+[2025-10-10 13:28:09,276] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.4542319774627686
+[2025-10-10 13:28:10,984] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.7078406810760498
+[2025-10-10 13:28:10,984] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.85it/s]
+
 75% 3/4 [00:02<00:00,  1.33it/s]
+
                                     
+
                                 
+
 30% 162/549 [16:25<29:40,  4.60s/it]
+
+
                                 
 30% 163/549 [16:29<52:53,  8.22s/it]
 30% 164/549 [16:33<44:20,  6.91s/it]
 30% 165/549 [16:37<38:21,  5.99s/it]
 30% 166/549 [16:41<34:12,  5.36s/it]
 30% 167/549 [16:44<31:21,  4.92s/it]
 31% 168/549 [16:48<29:18,  4.62s/it][2025-10-10 13:28:40,029] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 13:28:42,579] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2564804553985596
+[2025-10-10 13:28:43,831] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2514879703521729
+[2025-10-10 13:28:45,090] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2588951587677002
+[2025-10-10 13:28:46,762] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.6718604564666748
+[2025-10-10 13:28:46,762] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.85it/s]
+
 75% 3/4 [00:02<00:00,  1.33it/s]
+
                                     
+
                                 
+
 31% 168/549 [17:01<29:18,  4.62s/it]
+
+
                                 
 31% 169/549 [17:05<52:01,  8.21s/it]
 31% 170/549 [17:09<43:37,  6.91s/it]
 31% 171/549 [17:13<37:46,  6.00s/it]
 31% 172/549 [17:17<33:43,  5.37s/it]
 32% 173/549 [17:21<30:52,  4.93s/it]
 32% 174/549 [17:24<28:52,  4.62s/it][2025-10-10 13:29:16,068] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 13:29:18,702] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.3219468593597412
+[2025-10-10 13:29:20,003] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.301271915435791
+[2025-10-10 13:29:21,280] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2761008739471436
+[2025-10-10 13:29:22,700] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.4196476936340332
+[2025-10-10 13:29:22,700] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.85it/s]
+
 75% 3/4 [00:02<00:00,  1.33it/s]
+
                                     
+
                                 
+
 32% 174/549 [17:37<28:52,  4.62s/it]
+
+
                                 
 32% 175/549 [17:41<51:37,  8.28s/it]
 32% 176/549 [17:45<43:14,  6.96s/it]
 32% 177/549 [17:49<37:23,  6.03s/it]
 32% 178/549 [17:53<33:19,  5.39s/it]
 33% 179/549 [17:57<30:29,  4.94s/it]
 33% 180/549 [18:01<28:28,  4.63s/it][2025-10-10 13:29:52,329] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 13:29:54,842] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2502317428588867
+[2025-10-10 13:29:56,106] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2638275623321533
+[2025-10-10 13:29:57,362] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.255711317062378
+[2025-10-10 13:29:58,622] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.260202169418335
+[2025-10-10 13:29:58,622] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.85it/s]
+
 75% 3/4 [00:02<00:00,  1.33it/s]
+
                                     
+
                                 
+
 33% 180/549 [18:13<28:28,  4.63s/it]
+
+
                                 
 33% 181/549 [18:17<49:55,  8.14s/it]
 33% 182/549 [18:21<41:59,  6.86s/it]
 33% 183/549 [18:25<36:37,  6.00s/it][2025-10-10 13:30:16,541] [INFO] [axolotl.core.trainers.base._save:671] [PID:24741] Saving model checkpoint to ./qlora-out/checkpoint-183

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,21 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:15872f81ba0df2910609edb27c17b1b5641f9f269d4eba7fe013ab826ef0c746
+size 7313