config.json:   0%|                                                                                                       | 0.00/956 [00:00<?, ?B/s]
config.json: 100%|████████████████████████████████████████████████████████████████████████████████████████████████| 956/956 [00:00<00:00, 3.05MB/s]
[2026-04-05 19:19:08,110] [DEBUG] [axolotl.utils.config.log_gpu_memory_usage:127] [PID:2300] baseline 0.000GB ()
[2026-04-05 19:19:08,111] [INFO] [axolotl.cli.config.load_cfg:248] [PID:2300] config:
{
  "activation_offloading": false,
  "adapter": "lora",
  "axolotl_config_path": "config.yaml",
  "base_model": "unsloth/Meta-Llama-3.1-8B-Instruct",
  "base_model_config": "unsloth/Meta-Llama-3.1-8B-Instruct",
  "batch_size": 16,
  "bf16": true,
  "capabilities": {
    "bf16": true,
    "compute_capability": "sm_90",
    "fp8": false,
    "n_gpu": 1,
    "n_node": 1
  },
  "context_parallel_size": 1,
  "dataloader_num_workers": 1,
  "dataloader_pin_memory": true,
  "dataloader_prefetch_factor": 256,
  "dataset_processes": 26,
  "datasets": [
    {
      "chat_template": "tokenizer_default",
      "field_messages": "messages",
      "message_property_mappings": {
        "content": "content",
        "role": "role"
      },
      "path": "mx003/cve",
      "trust_remote_code": false,
      "type": "chat_template"
    }
  ],
  "ddp": false,
  "device": "cuda:0",
  "dion_rank_fraction": 1.0,
  "dion_rank_multiple_of": 1,
  "env_capabilities": {
    "torch_version": "2.7.1"
  },
  "eval_batch_size": 4,
  "eval_causal_lm_metrics": [
    "sacrebleu",
    "comet",
    "ter",
    "chrf"
  ],
  "eval_max_new_tokens": 128,
  "eval_table_size": 0,
  "experimental_skip_move_to_device": true,
  "fp16": false,
  "gradient_accumulation_steps": 4,
  "gradient_checkpointing": false,
  "group_by_length": true,
  "include_tkps": true,
  "is_llama_derived_model": true,
  "learning_rate": 0.0002,
  "lisa_layers_attribute": "model.layers",
  "load_best_model_at_end": false,
  "load_in_4bit": false,
  "load_in_8bit": false,
  "local_rank": 0,
  "lora_alpha": 64,
  "lora_dropout": 0.05,
  "lora_r": 32,
  "lora_target_modules": [
    "q_proj",
    "v_proj",
    "k_proj",
    "o_proj",
    "gate_proj",
    "down_proj",
    "up_proj"
  ],
  "loraplus_lr_embedding": 1e-06,
  "lr_scheduler": "cosine",
  "mean_resizing_embeddings": false,
  "micro_batch_size": 4,
  "model_config_type": "llama",
  "num_epochs": 3.0,
  "optimizer": "adamw_torch",
  "output_dir": "./outputs/mymodel",
  "pretrain_multipack_attn": true,
  "profiler_steps_start": 0,
  "qlora_sharded_model_loading": false,
  "ray_num_workers": 1,
  "resources_per_worker": {
    "GPU": 1
  },
  "sample_packing_bin_size": 200,
  "sample_packing_group_size": 100000,
  "save_only_model": false,
  "save_safetensors": true,
  "save_steps": 50,
  "sequence_len": 4096,
  "shuffle_before_merging_datasets": false,
  "shuffle_merged_datasets": true,
  "skip_prepare_dataset": false,
  "streaming_multipack_buffer_size": 10000,
  "strict": false,
  "tensor_parallel_size": 1,
  "tiled_mlp_use_original_mlp": true,
  "tokenizer_config": "unsloth/Meta-Llama-3.1-8B-Instruct",
  "tokenizer_save_jinja_files": true,
  "torch_dtype": "torch.bfloat16",
  "train_on_inputs": false,
  "trl": {
    "log_completions": false,
    "mask_truncated_completions": false,
    "ref_model_mixup_alpha": 0.9,
    "ref_model_sync_steps": 64,
    "scale_rewards": true,
    "sync_ref_model": false,
    "use_vllm": false,
    "vllm_server_host": "0.0.0.0",
    "vllm_server_port": 8000
  },
  "use_ray": false,
  "val_set_size": 0.0,
  "vllm": {
    "device": "auto",
    "dtype": "auto",
    "gpu_memory_utilization": 0.9,
    "host": "0.0.0.0",
    "port": 8000
  },
  "weight_decay": 0.0,
  "world_size": 1
}

tokenizer_config.json: 0.00B [00:00, ?B/s]
tokenizer_config.json: 55.5kB [00:00, 43.0MB/s]

tokenizer.json:   0%|                                                                                                  | 0.00/17.2M [00:00<?, ?B/s]
tokenizer.json: 100%|█████████████████████████████████████████████████████████████████████████████████████████| 17.2M/17.2M [00:02<00:00, 6.81MB/s]
tokenizer.json: 100%|█████████████████████████████████████████████████████████████████████████████████████████| 17.2M/17.2M [00:02<00:00, 6.81MB/s]

special_tokens_map.json:   0%|                                                                                           | 0.00/454 [00:00<?, ?B/s]
special_tokens_map.json: 100%|████████████████████████████████████████████████████████████████████████████████████| 454/454 [00:00<00:00, 1.30MB/s]
[2026-04-05 19:19:12,375] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:278] [PID:2300] EOS: 128009 / <|eot_id|>
[2026-04-05 19:19:12,376] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:279] [PID:2300] BOS: 128000 / <|begin_of_text|>
[2026-04-05 19:19:12,377] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:280] [PID:2300] PAD: 128004 / <|finetune_right_pad_id|>
[2026-04-05 19:19:12,377] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:281] [PID:2300] UNK: None / None
[2026-04-05 19:19:12,379] [INFO] [axolotl.utils.data.shared.load_preprocessed_dataset:476] [PID:2300] Unable to find prepared dataset in last_run_prepared/4818194e1a24026799335cd73096afba
[2026-04-05 19:19:12,380] [INFO] [axolotl.utils.data.sft._load_raw_datasets:320] [PID:2300] Loading raw datasets...
[2026-04-05 19:19:12,381] [WARNING] [axolotl.utils.data.sft._load_raw_datasets:322] [PID:2300] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`.
[2026-04-05 19:19:14,389] [INFO] [axolotl.utils.data.wrappers.get_dataset_wrapper:87] [PID:2300] Loading dataset: mx003/cve with base_type: chat_template and prompt_style: None
[2026-04-05 19:19:14,411] [INFO] [axolotl.prompt_strategies.chat_template.__call__:969] [PID:2300] Using chat template:
---
{{- bos_token }}
{%- if custom_tools is defined %}
    {%- set tools = custom_tools %}
{%- endif %}
{%- if not tools_in_user_message is defined %}
    {%- set tools_in_user_message = true %}
{%- endif %}
{%- if not date_string is defined %}
    {%- set date_string = "26 Jul 2024" %}
{%- endif %}
{%- if not tools is defined %}
    {%- set tools = none %}
{%- endif %}

{#- This block extracts the system message, so we can slot it into the right place. #}
{%- if messages[0]['role'] == 'system' %}
    {%- set system_message = messages[0]['content']|trim %}
    {%- set messages = messages[1:] %}
{%- else %}
    {%- set system_message = "" %}
{%- endif %}

{#- System message + builtin tools #}
{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
{%- if builtin_tools is defined or tools is not none %}
    {{- "Environment: ipython\n" }}
{%- endif %}
{%- if builtin_tools is defined %}
    {{- "Tools: " + builtin_tools | reject('equalto', 'code_interpreter') | join(", ") + "\n\n"}}
{%- endif %}
{{- "Cutting Knowledge Date: December 2023\n" }}
{{- "Today Date: " + date_string + "\n\n" }}
{%- if tools is not none and not tools_in_user_message %}
    {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
    {{- "Do not use variables.\n\n" }}
    {%- for t in tools %}
        {{- t | tojson(indent=4) }}
        {{- "\n\n" }}
    {%- endfor %}
{%- endif %}
{{- system_message }}
{{- "<|eot_id|>" }}

{#- Custom tools are passed in a user message with some extra guidance #}
{%- if tools_in_user_message and not tools is none %}
    {#- Extract the first user message so we can plug it in here #}
    {%- if messages | length != 0 %}
        {%- set first_user_message = messages[0]['content']|trim %}
        {%- set messages = messages[1:] %}
    {%- else %}
        {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
{%- endif %}
    {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
    {{- "Given the following functions, please respond with a JSON for a function call " }}
    {{- "with its proper arguments that best answers the given prompt.\n\n" }}
    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
    {{- "Do not use variables.\n\n" }}
    {%- for t in tools %}
        {{- t | tojson(indent=4) }}
        {{- "\n\n" }}
    {%- endfor %}
    {{- first_user_message + "<|eot_id|>"}}
{%- endif %}

{%- for message in messages %}
    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
    {%- elif 'tool_calls' in message %}
        {%- if not message.tool_calls|length == 1 %}
            {{- raise_exception("This model only supports single tool-calls at once!") }}
        {%- endif %}
        {%- set tool_call = message.tool_calls[0].function %}
        {%- if builtin_tools is defined and tool_call.name in builtin_tools %}
            {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
            {{- "<|python_tag|>" + tool_call.name + ".call(" }}
            {%- for arg_name, arg_val in tool_call.arguments | items %}
                {{- arg_name + '="' + arg_val + '"' }}
                {%- if not loop.last %}
                    {{- ", " }}
                {%- endif %}
                {%- endfor %}
            {{- ")" }}
        {%- else  %}
            {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
            {{- '{"name": "' + tool_call.name + '", ' }}
            {{- '"parameters": ' }}
            {{- tool_call.arguments | tojson }}
            {{- "}" }}
        {%- endif %}
        {%- if builtin_tools is defined %}
            {#- This means we're in ipython mode #}
            {{- "<|eom_id|>" }}
        {%- else %}
            {{- "<|eot_id|>" }}
        {%- endif %}
    {%- elif message.role == "tool" or message.role == "ipython" %}
        {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
        {%- if message.content is mapping or message.content is iterable %}
            {{- message.content | tojson }}
        {%- else %}
            {{- message.content }}
        {%- endif %}
        {{- "<|eot_id|>" }}
    {%- endif %}
{%- endfor %}
{%- if add_generation_prompt %}
    {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
{%- endif %}

---

Tokenizing Prompts (num_proc=26):   0%|                                                                             | 0/485 [00:00<?, ? examples/s]
Tokenizing Prompts (num_proc=26):   4%|██▋                                                                 | 19/485 [00:01<00:31, 14.69 examples/s]
Tokenizing Prompts (num_proc=26):   8%|█████▎                                                              | 38/485 [00:01<00:14, 29.91 examples/s]
Tokenizing Prompts (num_proc=26):  20%|█████████████▎                                                      | 95/485 [00:01<00:04, 85.06 examples/s]
Tokenizing Prompts (num_proc=26):  24%|███████████████▋                                                   | 114/485 [00:01<00:04, 90.67 examples/s]
Tokenizing Prompts (num_proc=26):  27%|██████████████████▎                                                | 133/485 [00:02<00:04, 78.49 examples/s]
Tokenizing Prompts (num_proc=26):  31%|████████████████████▉                                              | 152/485 [00:02<00:03, 87.21 examples/s]
Tokenizing Prompts (num_proc=26):  39%|██████████████████████████▏                                        | 190/485 [00:02<00:02, 99.01 examples/s]
Tokenizing Prompts (num_proc=26):  43%|████████████████████████████▍                                     | 209/485 [00:02<00:02, 111.29 examples/s]
Tokenizing Prompts (num_proc=26):  47%|███████████████████████████████▍                                   | 228/485 [00:03<00:03, 85.22 examples/s]
Tokenizing Prompts (num_proc=26):  51%|██████████████████████████████████                                 | 247/485 [00:03<00:02, 92.11 examples/s]
Tokenizing Prompts (num_proc=26):  55%|████████████████████████████████████▏                             | 266/485 [00:03<00:02, 101.64 examples/s]
Tokenizing Prompts (num_proc=26):  59%|██████████████████████████████████████▊                           | 285/485 [00:03<00:01, 112.55 examples/s]
Tokenizing Prompts (num_proc=26):  63%|█████████████████████████████████████████▎                        | 304/485 [00:03<00:01, 121.42 examples/s]
Tokenizing Prompts (num_proc=26):  67%|███████████████████████████████████████████▉                      | 323/485 [00:03<00:01, 124.05 examples/s]
Tokenizing Prompts (num_proc=26):  70%|██████████████████████████████████████████████▍                   | 341/485 [00:03<00:01, 126.06 examples/s]
Tokenizing Prompts (num_proc=26):  74%|████████████████████████████████████████████████▊                 | 359/485 [00:04<00:01, 122.11 examples/s]
Tokenizing Prompts (num_proc=26):  78%|███████████████████████████████████████████████████▎              | 377/485 [00:04<00:00, 119.17 examples/s]
Tokenizing Prompts (num_proc=26):  81%|█████████████████████████████████████████████████████▊            | 395/485 [00:04<00:00, 120.93 examples/s]
Tokenizing Prompts (num_proc=26):  89%|██████████████████████████████████████████████████████████▋       | 431/485 [00:04<00:00, 132.25 examples/s]
Tokenizing Prompts (num_proc=26):  93%|██████████████████████████████████████████████████████████████     | 449/485 [00:04<00:00, 99.80 examples/s]
Tokenizing Prompts (num_proc=26): 100%|██████████████████████████████████████████████████████████████████| 485/485 [00:05<00:00, 127.14 examples/s]
Tokenizing Prompts (num_proc=26): 100%|███████████████████████████████████████████████████████████████████| 485/485 [00:05<00:00, 90.28 examples/s]
[2026-04-05 19:19:20,047] [INFO] [axolotl.utils.data.utils.handle_long_seq_in_dataset:218] [PID:2300] min_input_len: 502
[2026-04-05 19:19:20,048] [INFO] [axolotl.utils.data.utils.handle_long_seq_in_dataset:220] [PID:2300] max_input_len: 8182

Dropping Long Sequences (>4096) (num_proc=26):   0%|                                                                | 0/485 [00:00<?, ? examples/s]
Dropping Long Sequences (>4096) (num_proc=26):   4%|██▏                                                    | 19/485 [00:00<00:09, 47.99 examples/s]
Dropping Long Sequences (>4096) (num_proc=26): 100%|█████████████████████████████████████████████████████| 485/485 [00:00<00:00, 765.61 examples/s]
[2026-04-05 19:19:20,790] [WARNING] [axolotl.utils.data.utils.handle_long_seq_in_dataset:260] [PID:2300] Dropped 26 samples from dataset

Saving the dataset (0/1 shards):   0%|                                                                              | 0/459 [00:00<?, ? examples/s]
Saving the dataset (1/1 shards): 100%|█████████████████████████████████████████████████████████████████| 459/459 [00:00<00:00, 12125.47 examples/s]
Saving the dataset (1/1 shards): 100%|█████████████████████████████████████████████████████████████████| 459/459 [00:00<00:00, 11705.11 examples/s]
[2026-04-05 19:19:20,957] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:404] [PID:2300] total_num_tokens: 697_096
[2026-04-05 19:19:20,967] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:422] [PID:2300] `total_supervised_tokens: 175_355`
[2026-04-05 19:19:20,968] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:520] [PID:2300] total_num_steps: 87
[2026-04-05 19:19:20,968] [INFO] [axolotl.utils.data.sft._prepare_standard_dataset:121] [PID:2300] Maximum number of steps set at 87
[2026-04-05 19:19:21,005] [DEBUG] [axolotl.train.setup_model_and_tokenizer:65] [PID:2300] Loading tokenizer... unsloth/Meta-Llama-3.1-8B-Instruct
[2026-04-05 19:19:21,824] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:278] [PID:2300] EOS: 128009 / <|eot_id|>
[2026-04-05 19:19:21,825] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:279] [PID:2300] BOS: 128000 / <|begin_of_text|>
[2026-04-05 19:19:21,826] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:280] [PID:2300] PAD: 128004 / <|finetune_right_pad_id|>
[2026-04-05 19:19:21,826] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:281] [PID:2300] UNK: None / None
[2026-04-05 19:19:21,826] [DEBUG] [axolotl.train.setup_model_and_tokenizer:74] [PID:2300] Loading model
[2026-04-05 19:19:21,950] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_evaluation_loop:87] [PID:2300] Patched Trainer.evaluation_loop with nanmean loss calculation
[2026-04-05 19:19:21,952] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_maybe_log_save_evaluate:138] [PID:2300] Patched Trainer._maybe_log_save_evaluate with nanmean loss calculation

model.safetensors.index.json: 0.00B [00:00, ?B/s]
model.safetensors.index.json: 23.9kB [00:00, 32.0MB/s]

model-00001-of-00004.safetensors:   0%|                                                                                | 0.00/4.98G [00:00<?, ?B/s]
model-00001-of-00004.safetensors:   0%|                                                                   | 8.13k/4.98G [00:02<474:11:25, 2.92kB/s]
model-00001-of-00004.safetensors:   0%|                                                                      | 2.23M/4.98G [00:03<1:47:41, 770kB/s]
model-00001-of-00004.safetensors:   0%|                                                                       | 5.15M/4.98G [00:04<59:54, 1.38MB/s]
model-00001-of-00004.safetensors:   0%|                                                                     | 5.41M/4.98G [00:05<1:01:01, 1.36MB/s]
model-00001-of-00004.safetensors:   0%|▏                                                                      | 13.7M/4.98G [00:05<18:50, 4.39MB/s]
model-00001-of-00004.safetensors:   0%|▏                                                                      | 16.2M/4.98G [00:06<17:03, 4.85MB/s]
model-00001-of-00004.safetensors:   1%|▍                                                                      | 28.4M/4.98G [00:06<06:56, 11.9MB/s]
model-00001-of-00004.safetensors:   1%|▋                                                                      | 48.2M/4.98G [00:06<03:04, 26.8MB/s]
model-00001-of-00004.safetensors:   1%|▊                                                                      | 60.2M/4.98G [00:06<03:11, 25.7MB/s]
model-00001-of-00004.safetensors:   1%|█                                                                      | 74.5M/4.98G [00:07<03:56, 20.7MB/s]
model-00001-of-00004.safetensors:   2%|█▍                                                                     | 96.5M/4.98G [00:08<03:00, 27.1MB/s]
model-00001-of-00004.safetensors:   2%|█▌                                                                      | 105M/4.98G [00:08<02:50, 28.5MB/s]
model-00001-of-00004.safetensors:   2%|█▌                                                                      | 112M/4.98G [00:09<04:21, 18.6MB/s]
model-00001-of-00004.safetensors:   2%|█▋                                                                      | 120M/4.98G [00:09<03:48, 21.3MB/s]
model-00001-of-00004.safetensors:   3%|█▉                                                                      | 135M/4.98G [00:10<02:45, 29.3MB/s]
model-00001-of-00004.safetensors:   3%|██                                                                      | 143M/4.98G [00:10<03:25, 23.5MB/s]
model-00001-of-00004.safetensors:   3%|██▎                                                                     | 163M/4.98G [00:11<03:20, 24.1MB/s]
model-00001-of-00004.safetensors:   3%|██▍                                                                     | 171M/4.98G [00:12<05:25, 14.8MB/s]
model-00001-of-00004.safetensors:   4%|██▌                                                                     | 180M/4.98G [00:13<04:56, 16.2MB/s]
model-00001-of-00004.safetensors:   4%|██▋                                                                     | 188M/4.98G [00:13<03:57, 20.2MB/s]
model-00001-of-00004.safetensors:   4%|██▉                                                                     | 200M/4.98G [00:13<03:05, 25.8MB/s]
model-00001-of-00004.safetensors:   4%|███▏                                                                    | 222M/4.98G [00:13<02:31, 31.5MB/s]
model-00001-of-00004.safetensors:   5%|███▊                                                                    | 262M/4.98G [00:14<01:15, 62.3MB/s]
model-00001-of-00004.safetensors:   6%|███▉                                                                    | 276M/4.98G [00:14<01:26, 54.4MB/s]
model-00001-of-00004.safetensors:   6%|████▏                                                                   | 289M/4.98G [00:16<04:11, 18.7MB/s]
model-00001-of-00004.safetensors:   6%|████▋                                                                   | 323M/4.98G [00:18<04:29, 17.3MB/s]
model-00001-of-00004.safetensors:   7%|████▊                                                                   | 336M/4.98G [00:19<04:10, 18.5MB/s]
model-00001-of-00004.safetensors:   7%|█████▎                                                                  | 369M/4.98G [00:19<02:41, 28.5MB/s]
model-00001-of-00004.safetensors:   8%|█████▍                                                                  | 380M/4.98G [00:19<02:30, 30.6MB/s]
model-00001-of-00004.safetensors:   8%|█████▋                                                                  | 392M/4.98G [00:20<03:00, 25.4MB/s]
model-00001-of-00004.safetensors:   8%|█████▉                                                                  | 413M/4.98G [00:20<02:20, 32.5MB/s]
model-00001-of-00004.safetensors:   9%|██████▎                                                                 | 435M/4.98G [00:21<02:33, 29.5MB/s]
model-00001-of-00004.safetensors:   9%|██████▋                                                                 | 460M/4.98G [00:22<01:48, 41.8MB/s]
model-00001-of-00004.safetensors:  10%|██████▉                                                                 | 477M/4.98G [00:22<02:20, 32.1MB/s]
model-00001-of-00004.safetensors:  10%|███████▍                                                                | 511M/4.98G [00:26<04:30, 16.5MB/s]
model-00001-of-00004.safetensors:  11%|███████▊                                                                | 544M/4.98G [00:27<03:52, 19.1MB/s]
model-00001-of-00004.safetensors:  12%|████████▌                                                               | 590M/4.98G [00:27<02:22, 30.8MB/s]
model-00001-of-00004.safetensors:  13%|█████████▌                                                              | 657M/4.98G [00:28<01:41, 42.4MB/s]
model-00001-of-00004.safetensors:  14%|██████████▎                                                             | 710M/4.98G [00:29<01:37, 43.6MB/s]
model-00001-of-00004.safetensors:  16%|███████████▏                                                            | 777M/4.98G [00:31<01:31, 45.7MB/s]
model-00001-of-00004.safetensors:  17%|████████████                                                            | 836M/4.98G [00:31<01:04, 63.8MB/s]
model-00001-of-00004.safetensors:  18%|████████████▊                                                           | 886M/4.98G [00:31<00:52, 78.2MB/s]
model-00001-of-00004.safetensors:  19%|█████████████▉                                                           | 953M/4.98G [00:32<00:39, 101MB/s]
model-00001-of-00004.safetensors:  20%|██████████████▋                                                         | 1.02G/4.98G [00:32<00:36, 108MB/s]
model-00001-of-00004.safetensors:  22%|███████████████▍                                                       | 1.08G/4.98G [00:33<00:42, 91.6MB/s]
model-00001-of-00004.safetensors:  23%|████████████████▍                                                      | 1.15G/4.98G [00:34<00:39, 95.8MB/s]
model-00001-of-00004.safetensors:  24%|█████████████████▋                                                      | 1.22G/4.98G [00:34<00:33, 114MB/s]
model-00001-of-00004.safetensors:  26%|██████████████████▌                                                     | 1.29G/4.98G [00:34<00:25, 146MB/s]
model-00001-of-00004.safetensors:  27%|███████████████████▌                                                    | 1.35G/4.98G [00:34<00:22, 159MB/s]
model-00001-of-00004.safetensors:  29%|████████████████████▌                                                   | 1.42G/4.98G [00:35<00:23, 150MB/s]
model-00001-of-00004.safetensors:  30%|█████████████████████▌                                                  | 1.49G/4.98G [00:36<00:31, 111MB/s]
model-00001-of-00004.safetensors:  31%|█████████████████████▋                                                 | 1.52G/4.98G [00:37<00:40, 85.8MB/s]
model-00001-of-00004.safetensors:  32%|███████████████████████                                                 | 1.59G/4.98G [00:37<00:30, 111MB/s]
model-00001-of-00004.safetensors:  33%|███████████████████████▉                                                | 1.66G/4.98G [00:37<00:23, 144MB/s]
model-00001-of-00004.safetensors:  34%|████████████████████████▌                                               | 1.69G/4.98G [00:38<00:24, 133MB/s]
model-00001-of-00004.safetensors:  35%|█████████████████████████▍                                              | 1.76G/4.98G [00:38<00:21, 152MB/s]
model-00001-of-00004.safetensors:  36%|█████████████████████████▊                                             | 1.81G/4.98G [00:39<00:38, 82.6MB/s]
model-00001-of-00004.safetensors:  37%|██████████████████████████▏                                            | 1.84G/4.98G [00:40<00:37, 83.0MB/s]
model-00001-of-00004.safetensors:  38%|███████████████████████████▌                                            | 1.90G/4.98G [00:40<00:29, 106MB/s]
model-00001-of-00004.safetensors:  40%|█████████████████████████████                                           | 2.01G/4.98G [00:40<00:18, 157MB/s]
model-00001-of-00004.safetensors:  41%|█████████████████████████████▋                                          | 2.05G/4.98G [00:40<00:16, 182MB/s]
model-00001-of-00004.safetensors:  43%|██████████████████████████████▋                                         | 2.12G/4.98G [00:41<00:21, 135MB/s]
model-00001-of-00004.safetensors:  44%|███████████████████████████████▋                                        | 2.19G/4.98G [00:41<00:19, 145MB/s]
model-00001-of-00004.safetensors:  45%|████████████████████████████████▋                                       | 2.26G/4.98G [00:42<00:16, 164MB/s]
model-00001-of-00004.safetensors:  47%|█████████████████████████████████▌                                      | 2.32G/4.98G [00:42<00:15, 171MB/s]
model-00001-of-00004.safetensors:  48%|██████████████████████████████████                                     | 2.39G/4.98G [00:44<00:28, 91.0MB/s]
model-00001-of-00004.safetensors:  49%|██████████████████████████████████▌                                    | 2.42G/4.98G [00:44<00:28, 89.2MB/s]
model-00001-of-00004.safetensors:  50%|███████████████████████████████████▌                                   | 2.49G/4.98G [00:45<00:30, 81.9MB/s]
model-00001-of-00004.safetensors:  51%|████████████████████████████████████▍                                  | 2.56G/4.98G [00:46<00:32, 75.5MB/s]
model-00001-of-00004.safetensors:  53%|█████████████████████████████████████▍                                 | 2.63G/4.98G [00:47<00:37, 62.2MB/s]
model-00001-of-00004.safetensors:  54%|██████████████████████████████████████▍                                | 2.69G/4.98G [00:48<00:27, 83.2MB/s]
model-00001-of-00004.safetensors:  55%|███████████████████████████████████████▎                               | 2.76G/4.98G [00:48<00:22, 97.9MB/s]
model-00001-of-00004.safetensors:  57%|████████████████████████████████████████▉                               | 2.83G/4.98G [00:49<00:19, 110MB/s]
model-00001-of-00004.safetensors:  58%|█████████████████████████████████████████▊                              | 2.89G/4.98G [00:49<00:16, 127MB/s]
model-00001-of-00004.safetensors:  59%|██████████████████████████████████████████▊                             | 2.96G/4.98G [00:49<00:12, 162MB/s]
model-00001-of-00004.safetensors:  61%|███████████████████████████████████████████▏                           | 3.03G/4.98G [00:51<00:29, 67.1MB/s]
model-00001-of-00004.safetensors:  62%|████████████████████████████████████████████▏                          | 3.09G/4.98G [00:52<00:25, 73.8MB/s]
model-00001-of-00004.safetensors:  64%|█████████████████████████████████████████████                          | 3.16G/4.98G [00:54<00:35, 50.5MB/s]
model-00001-of-00004.safetensors:  65%|██████████████████████████████████████████████                         | 3.23G/4.98G [00:55<00:26, 65.5MB/s]
model-00001-of-00004.safetensors:  66%|██████████████████████████████████████████████▋                        | 3.27G/4.98G [00:55<00:25, 67.7MB/s]
model-00001-of-00004.safetensors:  67%|███████████████████████████████████████████████▌                       | 3.34G/4.98G [00:59<00:44, 36.6MB/s]
model-00001-of-00004.safetensors:  68%|████████████████████████████████████████████████▌                      | 3.40G/4.98G [01:02<00:51, 30.4MB/s]
model-00001-of-00004.safetensors:  70%|█████████████████████████████████████████████████▌                     | 3.47G/4.98G [01:03<00:44, 33.8MB/s]
model-00001-of-00004.safetensors:  71%|██████████████████████████████████████████████████▍                    | 3.54G/4.98G [01:05<00:39, 36.4MB/s]
model-00001-of-00004.safetensors:  72%|███████████████████████████████████████████████████▍                   | 3.61G/4.98G [01:06<00:34, 40.0MB/s]
model-00001-of-00004.safetensors:  74%|████████████████████████████████████████████████████▎                  | 3.66G/4.98G [01:08<00:37, 35.3MB/s]
model-00001-of-00004.safetensors:  75%|█████████████████████████████████████████████████████▏                 | 3.73G/4.98G [01:08<00:25, 49.3MB/s]
model-00001-of-00004.safetensors:  76%|██████████████████████████████████████████████████████▏                | 3.80G/4.98G [01:10<00:22, 53.0MB/s]
model-00001-of-00004.safetensors:  78%|███████████████████████████████████████████████████████▏               | 3.87G/4.98G [01:10<00:16, 67.3MB/s]
model-00001-of-00004.safetensors:  79%|████████████████████████████████████████████████████████               | 3.93G/4.98G [01:11<00:17, 58.0MB/s]
model-00001-of-00004.safetensors:  80%|█████████████████████████████████████████████████████████              | 4.00G/4.98G [01:13<00:16, 58.9MB/s]
model-00001-of-00004.safetensors:  82%|██████████████████████████████████████████████████████████             | 4.07G/4.98G [01:13<00:12, 71.3MB/s]
model-00001-of-00004.safetensors:  83%|██████████████████████████████████████████████████████████▋            | 4.11G/4.98G [01:13<00:10, 80.9MB/s]
model-00001-of-00004.safetensors:  84%|████████████████████████████████████████████████████████████▍           | 4.18G/4.98G [01:13<00:07, 110MB/s]
model-00001-of-00004.safetensors:  85%|████████████████████████████████████████████████████████████▍          | 4.24G/4.98G [01:16<00:12, 60.2MB/s]
model-00001-of-00004.safetensors:  87%|█████████████████████████████████████████████████████████████▍         | 4.31G/4.98G [01:16<00:09, 69.0MB/s]
model-00001-of-00004.safetensors:  88%|██████████████████████████████████████████████████████████████▍        | 4.37G/4.98G [01:17<00:07, 84.7MB/s]
model-00001-of-00004.safetensors:  89%|███████████████████████████████████████████████████████████████▎       | 4.44G/4.98G [01:17<00:05, 95.9MB/s]
model-00001-of-00004.safetensors:  91%|█████████████████████████████████████████████████████████████████▏      | 4.51G/4.98G [01:17<00:03, 124MB/s]
model-00001-of-00004.safetensors:  92%|██████████████████████████████████████████████████████████████████▏     | 4.57G/4.98G [01:18<00:02, 151MB/s]
model-00001-of-00004.safetensors:  93%|██████████████████████████████████████████████████████████████████▏    | 4.64G/4.98G [01:21<00:06, 49.2MB/s]
model-00001-of-00004.safetensors:  95%|███████████████████████████████████████████████████████████████████▏   | 4.71G/4.98G [01:23<00:05, 44.7MB/s]
model-00001-of-00004.safetensors:  96%|████████████████████████████████████████████████████████████████████▏  | 4.78G/4.98G [01:28<00:07, 26.0MB/s]
model-00001-of-00004.safetensors:  97%|█████████████████████████████████████████████████████████████████████  | 4.84G/4.98G [01:31<00:05, 24.6MB/s]
model-00001-of-00004.safetensors:  99%|██████████████████████████████████████████████████████████████████████ | 4.91G/4.98G [01:32<00:02, 29.2MB/s]
model-00001-of-00004.safetensors:  99%|██████████████████████████████████████████████████████████████████████ | 4.91G/4.98G [01:44<00:02, 29.2MB/s]
model-00001-of-00004.safetensors: 100%|███████████████████████████████████████████████████████████████████████| 4.98G/4.98G [01:45<00:00, 12.4MB/s]
model-00001-of-00004.safetensors: 100%|███████████████████████████████████████████████████████████████████████| 4.98G/4.98G [01:45<00:00, 47.2MB/s]

model-00002-of-00004.safetensors:   0%|                                                                                | 0.00/5.00G [00:00<?, ?B/s]
model-00002-of-00004.safetensors:   0%|                                                                   | 8.13k/5.00G [00:04<854:04:44, 1.63kB/s]
model-00002-of-00004.safetensors:   0%|                                                                     | 5.45M/5.00G [00:06<1:20:30, 1.03MB/s]
model-00002-of-00004.safetensors:   1%|▍                                                                      | 28.4M/5.00G [00:07<14:53, 5.56MB/s]
model-00002-of-00004.safetensors:   1%|▌                                                                      | 36.7M/5.00G [00:08<12:03, 6.86MB/s]
model-00002-of-00004.safetensors:   1%|▌                                                                      | 43.6M/5.00G [00:08<09:28, 8.72MB/s]
model-00002-of-00004.safetensors:   1%|▉                                                                      | 63.9M/5.00G [00:09<06:20, 13.0MB/s]
model-00002-of-00004.safetensors:   1%|█                                                                      | 72.6M/5.00G [00:09<05:16, 15.6MB/s]
model-00002-of-00004.safetensors:   2%|█▏                                                                     | 80.7M/5.00G [00:10<04:49, 17.0MB/s]
model-00002-of-00004.safetensors:   2%|█▎                                                                     | 91.3M/5.00G [00:10<04:22, 18.7MB/s]
model-00002-of-00004.safetensors:   2%|█▋                                                                      | 114M/5.00G [00:10<02:27, 33.1MB/s]
model-00002-of-00004.safetensors:   2%|█▊                                                                      | 122M/5.00G [00:11<03:04, 26.5MB/s]
model-00002-of-00004.safetensors:   3%|██                                                                      | 140M/5.00G [00:11<02:14, 36.1MB/s]
model-00002-of-00004.safetensors:   3%|██▏                                                                     | 149M/5.00G [00:11<02:37, 30.7MB/s]
model-00002-of-00004.safetensors:   3%|██▎                                                                     | 157M/5.00G [00:12<02:34, 31.4MB/s]
model-00002-of-00004.safetensors:   3%|██▍                                                                     | 165M/5.00G [00:12<03:26, 23.4MB/s]
model-00002-of-00004.safetensors:   3%|██▌                                                                     | 174M/5.00G [00:13<03:25, 23.5MB/s]
model-00002-of-00004.safetensors:   4%|██▊                                                                     | 194M/5.00G [00:13<03:07, 25.6MB/s]
model-00002-of-00004.safetensors:   4%|██▉                                                                     | 203M/5.00G [00:14<03:23, 23.6MB/s]
model-00002-of-00004.safetensors:   4%|███▏                                                                    | 218M/5.00G [00:14<03:05, 25.7MB/s]
model-00002-of-00004.safetensors:   5%|███▎                                                                    | 226M/5.00G [00:14<02:48, 28.4MB/s]
model-00002-of-00004.safetensors:   5%|███▍                                                                    | 242M/5.00G [00:15<02:16, 35.0MB/s]
model-00002-of-00004.safetensors:   5%|███▌                                                                    | 251M/5.00G [00:15<02:05, 37.8MB/s]
model-00002-of-00004.safetensors:   5%|███▊                                                                    | 266M/5.00G [00:15<01:56, 40.6MB/s]
model-00002-of-00004.safetensors:   6%|████                                                                    | 284M/5.00G [00:16<02:59, 26.2MB/s]
model-00002-of-00004.safetensors:   6%|████▎                                                                   | 301M/5.00G [00:17<03:25, 22.8MB/s]
model-00002-of-00004.safetensors:   6%|████▋                                                                   | 325M/5.00G [00:18<02:31, 30.9MB/s]
model-00002-of-00004.safetensors:   7%|████▊                                                                   | 333M/5.00G [00:18<02:50, 27.3MB/s]
model-00002-of-00004.safetensors:   7%|████▉                                                                   | 341M/5.00G [00:18<02:32, 30.5MB/s]
model-00002-of-00004.safetensors:   7%|█████▍                                                                  | 375M/5.00G [00:18<01:23, 55.5MB/s]
model-00002-of-00004.safetensors:   8%|█████▋                                                                  | 392M/5.00G [00:19<01:39, 46.1MB/s]
model-00002-of-00004.safetensors:   8%|█████▊                                                                  | 401M/5.00G [00:22<05:18, 14.4MB/s]
model-00002-of-00004.safetensors:   8%|█████▉                                                                  | 411M/5.00G [00:22<04:31, 16.9MB/s]
model-00002-of-00004.safetensors:   9%|██████▊                                                                 | 474M/5.00G [00:23<02:18, 32.6MB/s]
model-00002-of-00004.safetensors:  10%|███████                                                                 | 488M/5.00G [00:23<02:09, 34.9MB/s]
model-00002-of-00004.safetensors:  11%|████████▏                                                               | 565M/5.00G [00:25<02:02, 36.3MB/s]
model-00002-of-00004.safetensors:  12%|████████▍                                                               | 584M/5.00G [00:26<01:56, 37.8MB/s]
model-00002-of-00004.safetensors:  13%|█████████                                                               | 627M/5.00G [00:26<01:33, 46.9MB/s]
model-00002-of-00004.safetensors:  13%|█████████▏                                                              | 641M/5.00G [00:29<03:08, 23.1MB/s]
model-00002-of-00004.safetensors:  14%|██████████▏                                                             | 708M/5.00G [00:29<01:45, 40.6MB/s]
model-00002-of-00004.safetensors:  15%|██████████▌                                                             | 731M/5.00G [00:30<01:53, 37.7MB/s]
model-00002-of-00004.safetensors:  16%|███████████▍                                                            | 795M/5.00G [00:30<01:14, 56.4MB/s]
model-00002-of-00004.safetensors:  17%|████████████▍                                                           | 862M/5.00G [00:30<00:52, 78.4MB/s]
model-00002-of-00004.safetensors:  18%|████████████▉                                                           | 903M/5.00G [00:31<00:50, 80.7MB/s]
model-00002-of-00004.safetensors:  19%|█████████████▌                                                          | 944M/5.00G [00:31<00:43, 93.3MB/s]
model-00002-of-00004.safetensors:  19%|█████████████▊                                                          | 961M/5.00G [00:33<01:29, 45.3MB/s]
model-00002-of-00004.safetensors:  21%|██████████████▌                                                        | 1.03G/5.00G [00:34<01:12, 54.5MB/s]
model-00002-of-00004.safetensors:  22%|███████████████▌                                                       | 1.10G/5.00G [00:34<00:51, 75.8MB/s]
model-00002-of-00004.safetensors:  23%|████████████████                                                       | 1.13G/5.00G [00:34<00:45, 84.5MB/s]
model-00002-of-00004.safetensors:  23%|████████████████▍                                                      | 1.16G/5.00G [00:34<00:40, 95.3MB/s]
model-00002-of-00004.safetensors:  24%|█████████████████▍                                                      | 1.21G/5.00G [00:35<00:30, 124MB/s]
model-00002-of-00004.safetensors:  26%|██████████████████▎                                                     | 1.27G/5.00G [00:35<00:27, 133MB/s]
model-00002-of-00004.safetensors:  26%|██████████████████▋                                                     | 1.30G/5.00G [00:35<00:33, 110MB/s]
model-00002-of-00004.safetensors:  27%|███████████████████▋                                                    | 1.37G/5.00G [00:36<00:24, 150MB/s]
model-00002-of-00004.safetensors:  29%|████████████████████▋                                                   | 1.43G/5.00G [00:36<00:21, 169MB/s]
model-00002-of-00004.safetensors:  30%|█████████████████████▌                                                  | 1.50G/5.00G [00:36<00:17, 198MB/s]
model-00002-of-00004.safetensors:  31%|██████████████████████▌                                                 | 1.57G/5.00G [00:37<00:17, 196MB/s]
model-00002-of-00004.safetensors:  32%|███████████████████████▏                                                | 1.61G/5.00G [00:37<00:21, 160MB/s]
model-00002-of-00004.safetensors:  34%|████████████████████████▏                                               | 1.68G/5.00G [00:37<00:16, 196MB/s]
model-00002-of-00004.safetensors:  35%|█████████████████████████▏                                              | 1.75G/5.00G [00:37<00:14, 220MB/s]
model-00002-of-00004.safetensors:  36%|██████████████████████████                                              | 1.81G/5.00G [00:38<00:16, 191MB/s]
model-00002-of-00004.safetensors:  38%|███████████████████████████                                             | 1.88G/5.00G [00:38<00:13, 224MB/s]
model-00002-of-00004.safetensors:  39%|████████████████████████████▏                                           | 1.95G/5.00G [00:38<00:14, 216MB/s]
model-00002-of-00004.safetensors:  40%|█████████████████████████████                                           | 2.02G/5.00G [00:39<00:14, 207MB/s]
model-00002-of-00004.safetensors:  41%|█████████████████████████████▌                                          | 2.06G/5.00G [00:39<00:14, 200MB/s]
model-00002-of-00004.safetensors:  42%|██████████████████████████████▌                                         | 2.12G/5.00G [00:39<00:11, 241MB/s]
model-00002-of-00004.safetensors:  44%|███████████████████████████████▌                                        | 2.19G/5.00G [00:39<00:10, 271MB/s]
model-00002-of-00004.safetensors:  45%|████████████████████████████████▍                                       | 2.26G/5.00G [00:39<00:08, 316MB/s]
model-00002-of-00004.safetensors:  46%|█████████████████████████████████▍                                      | 2.32G/5.00G [00:40<00:07, 357MB/s]
model-00002-of-00004.safetensors:  48%|██████████████████████████████████▍                                     | 2.39G/5.00G [00:40<00:07, 359MB/s]
model-00002-of-00004.safetensors:  49%|███████████████████████████████████▍                                    | 2.46G/5.00G [00:40<00:07, 334MB/s]
model-00002-of-00004.safetensors:  50%|████████████████████████████████████▎                                   | 2.52G/5.00G [00:40<00:08, 300MB/s]
model-00002-of-00004.safetensors:  51%|████████████████████████████████████▉                                   | 2.57G/5.00G [00:40<00:07, 309MB/s]
model-00002-of-00004.safetensors:  52%|█████████████████████████████████████▍                                  | 2.60G/5.00G [00:40<00:07, 309MB/s]
model-00002-of-00004.safetensors:  53%|██████████████████████████████████████▍                                 | 2.67G/5.00G [00:41<00:10, 232MB/s]
model-00002-of-00004.safetensors:  54%|███████████████████████████████████████                                 | 2.71G/5.00G [00:41<00:10, 225MB/s]
model-00002-of-00004.safetensors:  56%|███████████████████████████████████████▉                                | 2.78G/5.00G [00:41<00:08, 257MB/s]
model-00002-of-00004.safetensors:  57%|████████████████████████████████████████▉                               | 2.84G/5.00G [00:41<00:07, 304MB/s]
model-00002-of-00004.safetensors:  58%|█████████████████████████████████████████▌                              | 2.89G/5.00G [00:42<00:06, 308MB/s]
model-00002-of-00004.safetensors:  59%|██████████████████████████████████████████▌                             | 2.95G/5.00G [00:42<00:09, 210MB/s]
model-00002-of-00004.safetensors:  61%|███████████████████████████████████████████▋                            | 3.04G/5.00G [00:42<00:07, 261MB/s]
model-00002-of-00004.safetensors:  62%|████████████████████████████████████████████▋                           | 3.10G/5.00G [00:42<00:06, 308MB/s]
model-00002-of-00004.safetensors:  63%|█████████████████████████████████████████████▎                          | 3.15G/5.00G [00:43<00:05, 320MB/s]
model-00002-of-00004.safetensors:  65%|██████████████████████████████████████████████▋                         | 3.25G/5.00G [00:43<00:07, 242MB/s]
model-00002-of-00004.safetensors:  66%|███████████████████████████████████████████████▎                        | 3.29G/5.00G [00:43<00:07, 235MB/s]
model-00002-of-00004.safetensors:  67%|████████████████████████████████████████████████▎                       | 3.36G/5.00G [00:44<00:08, 188MB/s]
model-00002-of-00004.safetensors:  69%|█████████████████████████████████████████████████▉                      | 3.47G/5.00G [00:44<00:06, 230MB/s]
model-00002-of-00004.safetensors:  71%|██████████████████████████████████████████████████▉                     | 3.53G/5.00G [00:44<00:05, 267MB/s]
model-00002-of-00004.safetensors:  72%|███████████████████████████████████████████████████▏                   | 3.60G/5.00G [00:47<00:16, 83.6MB/s]
model-00002-of-00004.safetensors:  73%|████████████████████████████████████████████████████                   | 3.67G/5.00G [00:49<00:26, 51.1MB/s]
model-00002-of-00004.safetensors:  75%|█████████████████████████████████████████████████████                  | 3.73G/5.00G [00:49<00:19, 64.9MB/s]
model-00002-of-00004.safetensors:  76%|█████████████████████████████████████████████████████▉                 | 3.80G/5.00G [00:51<00:19, 60.7MB/s]
model-00002-of-00004.safetensors:  77%|██████████████████████████████████████████████████████▉                | 3.87G/5.00G [00:52<00:20, 55.2MB/s]
model-00002-of-00004.safetensors:  79%|███████████████████████████████████████████████████████▉               | 3.94G/5.00G [00:52<00:14, 74.6MB/s]
model-00002-of-00004.safetensors:  80%|████████████████████████████████████████████████████████▊              | 4.00G/5.00G [00:55<00:19, 50.5MB/s]
model-00002-of-00004.safetensors:  81%|█████████████████████████████████████████████████████████▊             | 4.07G/5.00G [00:55<00:14, 63.8MB/s]
model-00002-of-00004.safetensors:  83%|██████████████████████████████████████████████████████████▊            | 4.14G/5.00G [00:55<00:10, 84.9MB/s]
model-00002-of-00004.safetensors:  84%|████████████████████████████████████████████████████████████▌           | 4.20G/5.00G [00:55<00:07, 110MB/s]
model-00002-of-00004.safetensors:  85%|████████████████████████████████████████████████████████████▋          | 4.27G/5.00G [00:57<00:09, 77.9MB/s]
model-00002-of-00004.safetensors:  87%|█████████████████████████████████████████████████████████████▌         | 4.34G/5.00G [00:59<00:13, 50.8MB/s]
model-00002-of-00004.safetensors:  88%|██████████████████████████████████████████████████████████████▌        | 4.41G/5.00G [01:00<00:09, 61.8MB/s]
model-00002-of-00004.safetensors:  89%|███████████████████████████████████████████████████████████████▌       | 4.47G/5.00G [01:00<00:07, 75.0MB/s]
model-00002-of-00004.safetensors:  91%|████████████████████████████████████████████████████████████████▍      | 4.54G/5.00G [01:01<00:06, 74.0MB/s]
model-00002-of-00004.safetensors:  92%|█████████████████████████████████████████████████████████████████▎     | 4.60G/5.00G [01:02<00:04, 87.1MB/s]
model-00002-of-00004.safetensors:  93%|██████████████████████████████████████████████████████████████████▏    | 4.66G/5.00G [01:04<00:05, 58.7MB/s]
model-00002-of-00004.safetensors:  95%|███████████████████████████████████████████████████████████████████▏   | 4.73G/5.00G [01:04<00:03, 70.0MB/s]
model-00002-of-00004.safetensors:  96%|████████████████████████████████████████████████████████████████████▏  | 4.80G/5.00G [01:04<00:02, 94.8MB/s]
model-00002-of-00004.safetensors:  97%|█████████████████████████████████████████████████████████████████████  | 4.87G/5.00G [01:06<00:02, 65.6MB/s]
model-00002-of-00004.safetensors:  99%|██████████████████████████████████████████████████████████████████████ | 4.93G/5.00G [01:07<00:01, 61.1MB/s]
model-00002-of-00004.safetensors: 100%|███████████████████████████████████████████████████████████████████████| 5.00G/5.00G [01:09<00:00, 50.0MB/s]
model-00002-of-00004.safetensors: 100%|███████████████████████████████████████████████████████████████████████| 5.00G/5.00G [01:09<00:00, 71.8MB/s]

model-00003-of-00004.safetensors:   0%|                                                                                | 0.00/4.92G [00:00<?, ?B/s]
model-00003-of-00004.safetensors:   0%|                                                                   | 11.2k/4.92G [00:02<286:09:32, 4.77kB/s]
model-00003-of-00004.safetensors:   0%|                                                                   | 22.0k/4.92G [00:03<192:47:13, 7.08kB/s]
model-00003-of-00004.safetensors:   0%|                                                                       | 8.26M/4.92G [00:05<36:23, 2.25MB/s]
model-00003-of-00004.safetensors:   0%|▎                                                                      | 23.6M/4.92G [00:05<11:29, 7.09MB/s]
model-00003-of-00004.safetensors:   1%|▋                                                                      | 47.9M/4.92G [00:05<05:03, 16.0MB/s]
model-00003-of-00004.safetensors:   1%|▊                                                                      | 54.7M/4.92G [00:06<04:38, 17.5MB/s]
model-00003-of-00004.safetensors:   1%|▉                                                                      | 62.4M/4.92G [00:06<04:14, 19.1MB/s]
model-00003-of-00004.safetensors:   1%|█                                                                      | 70.8M/4.92G [00:06<03:32, 22.8MB/s]
model-00003-of-00004.safetensors:   2%|█▏                                                                     | 79.0M/4.92G [00:07<05:53, 13.7MB/s]
model-00003-of-00004.safetensors:   2%|█▏                                                                     | 85.7M/4.92G [00:08<05:36, 14.3MB/s]
model-00003-of-00004.safetensors:   2%|█▎                                                                     | 90.7M/4.92G [00:09<07:00, 11.5MB/s]
model-00003-of-00004.safetensors:   2%|█▍                                                                     | 99.0M/4.92G [00:09<06:49, 11.8MB/s]
model-00003-of-00004.safetensors:   2%|█▋                                                                      | 113M/4.92G [00:10<04:49, 16.6MB/s]
model-00003-of-00004.safetensors:   3%|█▉                                                                      | 131M/4.92G [00:10<02:59, 26.7MB/s]
model-00003-of-00004.safetensors:   3%|██                                                                      | 143M/4.92G [00:11<03:46, 21.0MB/s]
model-00003-of-00004.safetensors:   3%|██▎                                                                     | 154M/4.92G [00:11<03:15, 24.4MB/s]
model-00003-of-00004.safetensors:   3%|██▍                                                                     | 163M/4.92G [00:11<03:26, 23.0MB/s]
model-00003-of-00004.safetensors:   4%|██▋                                                                     | 183M/4.92G [00:12<02:59, 26.3MB/s]
model-00003-of-00004.safetensors:   4%|██▊                                                                     | 191M/4.92G [00:12<02:54, 27.1MB/s]
model-00003-of-00004.safetensors:   4%|██▉                                                                     | 199M/4.92G [00:13<02:50, 27.7MB/s]
model-00003-of-00004.safetensors:   4%|███▏                                                                    | 214M/4.92G [00:14<04:14, 18.5MB/s]
model-00003-of-00004.safetensors:   5%|███▎                                                                    | 228M/4.92G [00:14<03:17, 23.7MB/s]
model-00003-of-00004.safetensors:   5%|███▋                                                                    | 253M/4.92G [00:15<02:30, 31.0MB/s]
model-00003-of-00004.safetensors:   6%|███▉                                                                    | 273M/4.92G [00:15<02:52, 27.0MB/s]
model-00003-of-00004.safetensors:   6%|████▎                                                                   | 290M/4.92G [00:16<02:28, 31.2MB/s]
model-00003-of-00004.safetensors:   6%|████▍                                                                   | 306M/4.92G [00:16<02:02, 37.6MB/s]
model-00003-of-00004.safetensors:   7%|████▋                                                                   | 324M/4.92G [00:16<01:44, 43.9MB/s]
model-00003-of-00004.safetensors:   7%|████▉                                                                   | 340M/4.92G [00:17<01:34, 48.3MB/s]
model-00003-of-00004.safetensors:   8%|█████▍                                                                  | 369M/4.92G [00:20<04:57, 15.3MB/s]
model-00003-of-00004.safetensors:   8%|█████▉                                                                  | 402M/4.92G [00:21<03:41, 20.3MB/s]
model-00003-of-00004.safetensors:   9%|██████▍                                                                 | 436M/4.92G [00:21<02:29, 30.0MB/s]
model-00003-of-00004.safetensors:  10%|██████▊                                                                 | 469M/4.92G [00:23<02:43, 27.2MB/s]
model-00003-of-00004.safetensors:  10%|███████▏                                                                | 487M/4.92G [00:24<03:30, 21.1MB/s]
model-00003-of-00004.safetensors:  11%|███████▊                                                                | 531M/4.92G [00:25<02:41, 27.2MB/s]
model-00003-of-00004.safetensors:  12%|████████▍                                                               | 573M/4.92G [00:26<02:01, 35.7MB/s]
model-00003-of-00004.safetensors:  12%|████████▉                                                               | 606M/4.92G [00:27<02:10, 33.1MB/s]
model-00003-of-00004.safetensors:  13%|█████████▏                                                              | 625M/4.92G [00:28<02:49, 25.3MB/s]
model-00003-of-00004.safetensors:  13%|█████████▍                                                              | 646M/4.92G [00:29<02:31, 28.2MB/s]
model-00003-of-00004.safetensors:  14%|█████████▊                                                              | 666M/4.92G [00:30<02:24, 29.4MB/s]
model-00003-of-00004.safetensors:  15%|██████████▍                                                             | 717M/4.92G [00:30<01:30, 46.6MB/s]
model-00003-of-00004.safetensors:  15%|███████████                                                             | 759M/4.92G [00:30<01:07, 61.8MB/s]
model-00003-of-00004.safetensors:  16%|███████████▊                                                            | 805M/4.92G [00:30<00:52, 78.9MB/s]
model-00003-of-00004.safetensors:  18%|████████████▉                                                            | 872M/4.92G [00:31<00:35, 113MB/s]
model-00003-of-00004.safetensors:  19%|█████████████▌                                                           | 912M/4.92G [00:31<00:28, 138MB/s]
model-00003-of-00004.safetensors:  19%|█████████████▉                                                          | 955M/4.92G [00:33<01:10, 55.9MB/s]
model-00003-of-00004.safetensors:  20%|██████████████▍                                                         | 988M/4.92G [00:34<01:26, 45.5MB/s]
model-00003-of-00004.safetensors:  21%|██████████████▉                                                        | 1.03G/4.92G [00:34<01:01, 63.0MB/s]
model-00003-of-00004.safetensors:  22%|███████████████▊                                                       | 1.09G/4.92G [00:35<01:12, 53.1MB/s]
model-00003-of-00004.safetensors:  24%|████████████████▋                                                      | 1.16G/4.92G [00:36<00:47, 79.8MB/s]
model-00003-of-00004.safetensors:  25%|█████████████████▋                                                     | 1.23G/4.92G [00:36<00:39, 93.6MB/s]
model-00003-of-00004.safetensors:  26%|██████████████████▋                                                    | 1.29G/4.92G [00:37<00:41, 88.2MB/s]
model-00003-of-00004.safetensors:  27%|███████████████████▍                                                   | 1.34G/4.92G [00:38<00:43, 82.0MB/s]
model-00003-of-00004.safetensors:  29%|████████████████████▎                                                  | 1.41G/4.92G [00:38<00:35, 98.9MB/s]
model-00003-of-00004.safetensors:  30%|█████████████████████▋                                                  | 1.48G/4.92G [00:38<00:31, 110MB/s]
model-00003-of-00004.safetensors:  31%|██████████████████████▏                                                 | 1.52G/4.92G [00:39<00:26, 127MB/s]
model-00003-of-00004.safetensors:  32%|███████████████████████▏                                                | 1.58G/4.92G [00:39<00:28, 118MB/s]
model-00003-of-00004.safetensors:  33%|███████████████████████▊                                                | 1.63G/4.92G [00:39<00:24, 134MB/s]
model-00003-of-00004.safetensors:  35%|████████████████████████▊                                               | 1.70G/4.92G [00:40<00:19, 164MB/s]
model-00003-of-00004.safetensors:  35%|█████████████████████████▍                                              | 1.74G/4.92G [00:40<00:17, 187MB/s]
model-00003-of-00004.safetensors:  37%|██████████████████████████▍                                             | 1.80G/4.92G [00:40<00:18, 169MB/s]
model-00003-of-00004.safetensors:  38%|███████████████████████████▍                                            | 1.87G/4.92G [00:41<00:17, 169MB/s]
model-00003-of-00004.safetensors:  39%|███████████████████████████▉                                            | 1.90G/4.92G [00:41<00:18, 160MB/s]
model-00003-of-00004.safetensors:  40%|████████████████████████████▉                                           | 1.97G/4.92G [00:41<00:14, 204MB/s]
model-00003-of-00004.safetensors:  41%|█████████████████████████████▊                                          | 2.04G/4.92G [00:41<00:11, 243MB/s]
model-00003-of-00004.safetensors:  43%|██████████████████████████████▊                                         | 2.11G/4.92G [00:42<00:23, 121MB/s]
model-00003-of-00004.safetensors:  44%|███████████████████████████████▊                                        | 2.17G/4.92G [00:43<00:17, 153MB/s]
model-00003-of-00004.safetensors:  46%|████████████████████████████████▊                                       | 2.24G/4.92G [00:44<00:24, 110MB/s]
model-00003-of-00004.safetensors:  47%|█████████████████████████████████▊                                      | 2.31G/4.92G [00:44<00:23, 110MB/s]
model-00003-of-00004.safetensors:  48%|██████████████████████████████████▍                                     | 2.35G/4.92G [00:44<00:19, 131MB/s]
model-00003-of-00004.safetensors:  49%|███████████████████████████████████▎                                    | 2.41G/4.92G [00:45<00:18, 135MB/s]
model-00003-of-00004.safetensors:  50%|████████████████████████████████████▎                                   | 2.48G/4.92G [00:45<00:16, 151MB/s]
model-00003-of-00004.safetensors:  51%|████████████████████████████████████▉                                   | 2.52G/4.92G [00:45<00:15, 150MB/s]
model-00003-of-00004.safetensors:  53%|█████████████████████████████████████▉                                  | 2.59G/4.92G [00:46<00:19, 117MB/s]
model-00003-of-00004.safetensors:  54%|██████████████████████████████████████▎                                | 2.65G/4.92G [00:47<00:23, 95.1MB/s]
model-00003-of-00004.safetensors:  55%|███████████████████████████████████████▊                                | 2.72G/4.92G [00:47<00:17, 125MB/s]
model-00003-of-00004.safetensors:  57%|████████████████████████████████████████▎                              | 2.79G/4.92G [00:48<00:21, 98.7MB/s]
model-00003-of-00004.safetensors:  58%|█████████████████████████████████████████▊                              | 2.86G/4.92G [00:49<00:16, 123MB/s]
model-00003-of-00004.safetensors:  59%|██████████████████████████████████████████▊                             | 2.92G/4.92G [00:49<00:12, 155MB/s]
model-00003-of-00004.safetensors:  61%|███████████████████████████████████████████▊                            | 2.99G/4.92G [00:49<00:10, 176MB/s]
model-00003-of-00004.safetensors:  62%|████████████████████████████████████████████▌                           | 3.04G/4.92G [00:49<00:09, 197MB/s]
model-00003-of-00004.safetensors:  63%|█████████████████████████████████████████████▌                          | 3.11G/4.92G [00:50<00:12, 143MB/s]
model-00003-of-00004.safetensors:  65%|██████████████████████████████████████████████▌                         | 3.18G/4.92G [00:50<00:09, 188MB/s]
model-00003-of-00004.safetensors:  66%|███████████████████████████████████████████████▌                        | 3.24G/4.92G [00:50<00:07, 221MB/s]
model-00003-of-00004.safetensors:  67%|████████████████████████████████████████████████▍                       | 3.31G/4.92G [00:51<00:09, 166MB/s]
model-00003-of-00004.safetensors:  69%|█████████████████████████████████████████████████▍                      | 3.37G/4.92G [00:51<00:07, 203MB/s]
model-00003-of-00004.safetensors:  70%|██████████████████████████████████████████████████▍                     | 3.44G/4.92G [00:51<00:07, 193MB/s]
model-00003-of-00004.safetensors:  71%|███████████████████████████████████████████████████▎                    | 3.51G/4.92G [00:52<00:09, 142MB/s]
model-00003-of-00004.safetensors:  73%|████████████████████████████████████████████████████▎                   | 3.57G/4.92G [00:52<00:09, 147MB/s]
model-00003-of-00004.safetensors:  74%|█████████████████████████████████████████████████████▎                  | 3.64G/4.92G [00:53<00:07, 172MB/s]
model-00003-of-00004.safetensors:  75%|██████████████████████████████████████████████████████▎                 | 3.71G/4.92G [00:53<00:08, 150MB/s]
model-00003-of-00004.safetensors:  77%|███████████████████████████████████████████████████████▎                | 3.78G/4.92G [00:54<00:07, 143MB/s]
model-00003-of-00004.safetensors:  78%|████████████████████████████████████████████████████████▎               | 3.84G/4.92G [00:54<00:06, 166MB/s]
model-00003-of-00004.safetensors:  80%|█████████████████████████████████████████████████████████▎              | 3.91G/4.92G [00:55<00:08, 122MB/s]
model-00003-of-00004.safetensors:  81%|██████████████████████████████████████████████████████████▏             | 3.98G/4.92G [00:55<00:07, 133MB/s]
model-00003-of-00004.safetensors:  82%|███████████████████████████████████████████████████████████▏            | 4.04G/4.92G [00:56<00:06, 142MB/s]
model-00003-of-00004.safetensors:  84%|████████████████████████████████████████████████████████████▏           | 4.11G/4.92G [00:56<00:05, 152MB/s]
model-00003-of-00004.safetensors:  85%|█████████████████████████████████████████████████████████████▏          | 4.18G/4.92G [00:57<00:05, 139MB/s]
model-00003-of-00004.safetensors:  86%|██████████████████████████████████████████████████████████████▏         | 4.25G/4.92G [00:57<00:04, 153MB/s]
model-00003-of-00004.safetensors:  88%|███████████████████████████████████████████████████████████████▏        | 4.31G/4.92G [00:57<00:03, 191MB/s]
model-00003-of-00004.safetensors:  89%|████████████████████████████████████████████████████████████████▏       | 4.38G/4.92G [00:58<00:04, 117MB/s]
model-00003-of-00004.safetensors:  90%|████████████████████████████████████████████████████████████████▏      | 4.45G/4.92G [01:00<00:06, 72.3MB/s]
model-00003-of-00004.safetensors:  92%|█████████████████████████████████████████████████████████████████▏     | 4.51G/4.92G [01:00<00:04, 90.5MB/s]
model-00003-of-00004.safetensors:  93%|███████████████████████████████████████████████████████████████████     | 4.58G/4.92G [01:01<00:03, 107MB/s]
model-00003-of-00004.safetensors:  95%|████████████████████████████████████████████████████████████████████    | 4.65G/4.92G [01:01<00:02, 125MB/s]
model-00003-of-00004.safetensors:  96%|████████████████████████████████████████████████████████████████████   | 4.71G/4.92G [01:03<00:02, 67.6MB/s]
model-00003-of-00004.safetensors:  97%|█████████████████████████████████████████████████████████████████████  | 4.78G/4.92G [01:05<00:02, 51.7MB/s]
model-00003-of-00004.safetensors:  99%|██████████████████████████████████████████████████████████████████████ | 4.85G/4.92G [01:10<00:02, 28.0MB/s]
model-00003-of-00004.safetensors: 100%|███████████████████████████████████████████████████████████████████████| 4.92G/4.92G [01:15<00:00, 21.5MB/s]
model-00003-of-00004.safetensors: 100%|███████████████████████████████████████████████████████████████████████| 4.92G/4.92G [01:15<00:00, 65.2MB/s]

model-00004-of-00004.safetensors:   0%|                                                                                | 0.00/1.17G [00:00<?, ?B/s]
model-00004-of-00004.safetensors:   0%|                                                                       | 224k/1.17G [00:01<2:12:53, 146kB/s]
model-00004-of-00004.safetensors:   0%|                                                                      | 336k/1.17G [00:05<6:35:46, 49.2kB/s]
model-00004-of-00004.safetensors:   1%|▊                                                                      | 13.4M/1.17G [00:07<07:19, 2.63MB/s]
model-00004-of-00004.safetensors:   2%|█▍                                                                     | 23.9M/1.17G [00:07<03:46, 5.06MB/s]
model-00004-of-00004.safetensors:   3%|██                                                                     | 33.9M/1.17G [00:08<02:52, 6.59MB/s]
model-00004-of-00004.safetensors:   5%|███▊                                                                   | 63.7M/1.17G [00:13<02:52, 6.40MB/s]
model-00004-of-00004.safetensors:  11%|████████                                                                | 131M/1.17G [00:17<01:41, 10.3MB/s]
model-00004-of-00004.safetensors:  16%|███████████▍                                                            | 185M/1.17G [00:19<01:06, 14.7MB/s]
model-00004-of-00004.safetensors:  22%|███████████████▌                                                        | 252M/1.17G [00:19<00:38, 23.6MB/s]
model-00004-of-00004.safetensors:  27%|███████████████████▋                                                    | 319M/1.17G [00:20<00:24, 34.8MB/s]
model-00004-of-00004.safetensors:  32%|███████████████████████                                                 | 373M/1.17G [00:20<00:17, 46.2MB/s]
model-00004-of-00004.safetensors:  38%|███████████████████████████▏                                            | 440M/1.17G [00:21<00:14, 50.7MB/s]
model-00004-of-00004.safetensors:  43%|███████████████████████████████▎                                        | 507M/1.17G [00:23<00:13, 47.6MB/s]
model-00004-of-00004.safetensors:  49%|███████████████████████████████████▍                                    | 575M/1.17G [00:24<00:12, 48.9MB/s]
model-00004-of-00004.safetensors:  54%|██████████████████████████████████████▉                                 | 632M/1.17G [00:25<00:09, 54.9MB/s]
model-00004-of-00004.safetensors:  60%|███████████████████████████████████████████                             | 699M/1.17G [00:25<00:06, 74.8MB/s]
model-00004-of-00004.safetensors:  66%|███████████████████████████████████████████████▏                        | 766M/1.17G [00:26<00:04, 91.6MB/s]
model-00004-of-00004.safetensors:  71%|███████████████████████████████████████████████████▎                    | 833M/1.17G [00:27<00:05, 63.2MB/s]
model-00004-of-00004.safetensors:  77%|███████████████████████████████████████████████████████▍                | 900M/1.17G [00:28<00:03, 78.4MB/s]
model-00004-of-00004.safetensors:  83%|███████████████████████████████████████████████████████████▌            | 967M/1.17G [00:31<00:04, 42.8MB/s]
model-00004-of-00004.safetensors:  89%|██████████████████████████████████████████████████████████████▊        | 1.03G/1.17G [00:32<00:02, 45.1MB/s]
model-00004-of-00004.safetensors:  94%|██████████████████████████████████████████████████████████████████▉    | 1.10G/1.17G [00:33<00:01, 57.9MB/s]
model-00004-of-00004.safetensors: 100%|███████████████████████████████████████████████████████████████████████| 1.17G/1.17G [00:37<00:00, 30.1MB/s]
model-00004-of-00004.safetensors: 100%|███████████████████████████████████████████████████████████████████████| 1.17G/1.17G [00:37<00:00, 30.9MB/s]

Loading checkpoint shards:   0%|                                                                                             | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████| 4/4 [00:00<00:00, 47.00it/s]

generation_config.json:   0%|                                                                                            | 0.00/239 [00:00<?, ?B/s]
generation_config.json: 100%|██████████████████████████████████████████████████████████████████████████████████████| 239/239 [00:00<00:00, 628kB/s]
[2026-04-05 19:24:15,254] [INFO] [axolotl.loaders.model._configure_embedding_dtypes:345] [PID:2300] Converting modules to torch.bfloat16
[2026-04-05 19:24:16,536] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:2300] Memory usage after model load 0.000GB ()
trainable params: 83,886,080 || all params: 8,114,147,328 || trainable%: 1.0338
[2026-04-05 19:24:17,207] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:2300] after adapters 0.000GB ()
[2026-04-05 19:24:26,689] [INFO] [axolotl.train.save_initial_configs:398] [PID:2300] Pre-saving adapter config to ./outputs/mymodel...
[2026-04-05 19:24:26,695] [INFO] [axolotl.train.save_initial_configs:402] [PID:2300] Pre-saving tokenizer to ./outputs/mymodel...
[2026-04-05 19:24:27,070] [INFO] [axolotl.train.save_initial_configs:407] [PID:2300] Pre-saving model config to ./outputs/mymodel...
[2026-04-05 19:24:27,078] [INFO] [axolotl.train.execute_training:196] [PID:2300] Starting trainer...

  0%|                                                                                                                       | 0/87 [00:00<?, ?it/s]Traceback (most recent call last):
  File "<frozen runpy>", line 198, in _run_module_as_main
  File "<frozen runpy>", line 88, in _run_code
  File "/workspace/axolotl/src/axolotl/cli/train.py", line 121, in <module>
    fire.Fire(do_cli)
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/fire/core.py", line 135, in Fire
    component_trace = _Fire(component, args, parsed_flag_args, context, name)
                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/fire/core.py", line 468, in _Fire
    component, remaining_args = _CallAndUpdateTrace(
                                ^^^^^^^^^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/fire/core.py", line 684, in _CallAndUpdateTrace
    component = fn(*varargs, **kwargs)
                ^^^^^^^^^^^^^^^^^^^^^^
  File "/workspace/axolotl/src/axolotl/cli/train.py", line 88, in do_cli
    return do_train(parsed_cfg, parsed_cli_args)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/workspace/axolotl/src/axolotl/cli/train.py", line 45, in do_train
    model, tokenizer, trainer = train(cfg=cfg, dataset_meta=dataset_meta)
                                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/workspace/axolotl/src/axolotl/train.py", line 573, in train
    execute_training(cfg, trainer, resume_from_checkpoint)
  File "/workspace/axolotl/src/axolotl/train.py", line 197, in execute_training
    trainer.train(resume_from_checkpoint=resume_from_checkpoint)
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/trainer.py", line 2325, in train
    return inner_training_loop(
           ^^^^^^^^^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/trainer.py", line 2674, in _inner_training_loop
    tr_loss_step = self.training_step(model, inputs, num_items_in_batch)
                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/workspace/axolotl/src/axolotl/core/trainers/mixins/activation_checkpointing.py", line 46, in training_step
    return super().training_step(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/trainer.py", line 4020, in training_step
    loss = self.compute_loss(model, inputs, num_items_in_batch=num_items_in_batch)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/workspace/axolotl/src/axolotl/core/trainers/base.py", line 367, in compute_loss
    return super().compute_loss(
           ^^^^^^^^^^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/trainer.py", line 4110, in compute_loss
    outputs = model(**inputs)
              ^^^^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1762, in _call_impl
    return forward_call(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/accelerate/utils/operations.py", line 818, in forward
    return model_forward(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/accelerate/utils/operations.py", line 806, in __call__
    return convert_to_fp32(self.model_forward(*args, **kwargs))
                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/torch/amp/autocast_mode.py", line 44, in decorate_autocast
    return func(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/peft/peft_model.py", line 1850, in forward
    return self.base_model(
           ^^^^^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1762, in _call_impl
    return forward_call(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/peft/tuners/tuners_utils.py", line 222, in forward
    return self.model.forward(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/utils/generic.py", line 918, in wrapper
    output = func(self, *args, **kwargs)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/models/llama/modeling_llama.py", line 459, in forward
    outputs: BaseModelOutputWithPast = self.model(
                                       ^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1762, in _call_impl
    return forward_call(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/utils/generic.py", line 1064, in wrapper
    outputs = func(self, *args, **kwargs)
              ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/models/llama/modeling_llama.py", line 395, in forward
    hidden_states = decoder_layer(
                    ^^^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/modeling_layers.py", line 94, in __call__
    return super().__call__(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1762, in _call_impl
    return forward_call(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/utils/deprecation.py", line 172, in wrapped_func
    return func(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/models/llama/modeling_llama.py", line 309, in forward
    hidden_states = self.mlp(hidden_states)
                    ^^^^^^^^^^^^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1762, in _call_impl
    return forward_call(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/models/llama/modeling_llama.py", line 155, in forward
    down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
                                           ^^^^^^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1762, in _call_impl
    return forward_call(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/peft/tuners/lora/layer.py", line 771, in forward
    result = result + lora_B(lora_A(dropout(x))) * scaling
                      ^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1762, in _call_impl
    return forward_call(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/torch/nn/modules/linear.py", line 125, in forward
    return F.linear(input, self.weight, self.bias)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 448.00 MiB. GPU 0 has a total capacity of 79.18 GiB of which 298.19 MiB is free. Including non-PyTorch memory, this process has 78.88 GiB memory in use. Of the allocated memory 77.93 GiB is allocated by PyTorch, and 298.73 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation.  See documentation for Memory Management  (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)

  0%|                                                                                                                       | 0/87 [00:02<?, ?it/s]