|
config.json: 0%| | 0.00/956 [00:00<?, ?B/s]
config.json: 100%|████████████████████████████████████████████████████████████████████████████████████████████████| 956/956 [00:00<00:00, 3.05MB/s] |
| [2026-04-05 19:19:08,110] [DEBUG] [axolotl.utils.config.log_gpu_memory_usage:127] [PID:2300] baseline 0.000GB () |
| [2026-04-05 19:19:08,111] [INFO] [axolotl.cli.config.load_cfg:248] [PID:2300] config: |
| { |
| "activation_offloading": false, |
| "adapter": "lora", |
| "axolotl_config_path": "config.yaml", |
| "base_model": "unsloth/Meta-Llama-3.1-8B-Instruct", |
| "base_model_config": "unsloth/Meta-Llama-3.1-8B-Instruct", |
| "batch_size": 16, |
| "bf16": true, |
| "capabilities": { |
| "bf16": true, |
| "compute_capability": "sm_90", |
| "fp8": false, |
| "n_gpu": 1, |
| "n_node": 1 |
| }, |
| "context_parallel_size": 1, |
| "dataloader_num_workers": 1, |
| "dataloader_pin_memory": true, |
| "dataloader_prefetch_factor": 256, |
| "dataset_processes": 26, |
| "datasets": [ |
| { |
| "chat_template": "tokenizer_default", |
| "field_messages": "messages", |
| "message_property_mappings": { |
| "content": "content", |
| "role": "role" |
| }, |
| "path": "mx003/cve", |
| "trust_remote_code": false, |
| "type": "chat_template" |
| } |
| ], |
| "ddp": false, |
| "device": "cuda:0", |
| "dion_rank_fraction": 1.0, |
| "dion_rank_multiple_of": 1, |
| "env_capabilities": { |
| "torch_version": "2.7.1" |
| }, |
| "eval_batch_size": 4, |
| "eval_causal_lm_metrics": [ |
| "sacrebleu", |
| "comet", |
| "ter", |
| "chrf" |
| ], |
| "eval_max_new_tokens": 128, |
| "eval_table_size": 0, |
| "experimental_skip_move_to_device": true, |
| "fp16": false, |
| "gradient_accumulation_steps": 4, |
| "gradient_checkpointing": false, |
| "group_by_length": true, |
| "include_tkps": true, |
| "is_llama_derived_model": true, |
| "learning_rate": 0.0002, |
| "lisa_layers_attribute": "model.layers", |
| "load_best_model_at_end": false, |
| "load_in_4bit": false, |
| "load_in_8bit": false, |
| "local_rank": 0, |
| "lora_alpha": 64, |
| "lora_dropout": 0.05, |
| "lora_r": 32, |
| "lora_target_modules": [ |
| "q_proj", |
| "v_proj", |
| "k_proj", |
| "o_proj", |
| "gate_proj", |
| "down_proj", |
| "up_proj" |
| ], |
| "loraplus_lr_embedding": 1e-06, |
| "lr_scheduler": "cosine", |
| "mean_resizing_embeddings": false, |
| "micro_batch_size": 4, |
| "model_config_type": "llama", |
| "num_epochs": 3.0, |
| "optimizer": "adamw_torch", |
| "output_dir": "./outputs/mymodel", |
| "pretrain_multipack_attn": true, |
| "profiler_steps_start": 0, |
| "qlora_sharded_model_loading": false, |
| "ray_num_workers": 1, |
| "resources_per_worker": { |
| "GPU": 1 |
| }, |
| "sample_packing_bin_size": 200, |
| "sample_packing_group_size": 100000, |
| "save_only_model": false, |
| "save_safetensors": true, |
| "save_steps": 50, |
| "sequence_len": 4096, |
| "shuffle_before_merging_datasets": false, |
| "shuffle_merged_datasets": true, |
| "skip_prepare_dataset": false, |
| "streaming_multipack_buffer_size": 10000, |
| "strict": false, |
| "tensor_parallel_size": 1, |
| "tiled_mlp_use_original_mlp": true, |
| "tokenizer_config": "unsloth/Meta-Llama-3.1-8B-Instruct", |
| "tokenizer_save_jinja_files": true, |
| "torch_dtype": "torch.bfloat16", |
| "train_on_inputs": false, |
| "trl": { |
| "log_completions": false, |
| "mask_truncated_completions": false, |
| "ref_model_mixup_alpha": 0.9, |
| "ref_model_sync_steps": 64, |
| "scale_rewards": true, |
| "sync_ref_model": false, |
| "use_vllm": false, |
| "vllm_server_host": "0.0.0.0", |
| "vllm_server_port": 8000 |
| }, |
| "use_ray": false, |
| "val_set_size": 0.0, |
| "vllm": { |
| "device": "auto", |
| "dtype": "auto", |
| "gpu_memory_utilization": 0.9, |
| "host": "0.0.0.0", |
| "port": 8000 |
| }, |
| "weight_decay": 0.0, |
| "world_size": 1 |
| } |
|
tokenizer_config.json: 0.00B [00:00, ?B/s]
tokenizer_config.json: 55.5kB [00:00, 43.0MB/s] |
|
tokenizer.json: 0%| | 0.00/17.2M [00:00<?, ?B/s]
tokenizer.json: 100%|█████████████████████████████████████████████████████████████████████████████████████████| 17.2M/17.2M [00:02<00:00, 6.81MB/s]
tokenizer.json: 100%|█████████████████████████████████████████████████████████████████████████████████████████| 17.2M/17.2M [00:02<00:00, 6.81MB/s] |
|
special_tokens_map.json: 0%| | 0.00/454 [00:00<?, ?B/s]
special_tokens_map.json: 100%|████████████████████████████████████████████████████████████████████████████████████| 454/454 [00:00<00:00, 1.30MB/s] |
| [2026-04-05 19:19:12,375] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:278] [PID:2300] EOS: 128009 / <|eot_id|> |
| [2026-04-05 19:19:12,376] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:279] [PID:2300] BOS: 128000 / <|begin_of_text|> |
| [2026-04-05 19:19:12,377] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:280] [PID:2300] PAD: 128004 / <|finetune_right_pad_id|> |
| [2026-04-05 19:19:12,377] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:281] [PID:2300] UNK: None / None |
| [2026-04-05 19:19:12,379] [INFO] [axolotl.utils.data.shared.load_preprocessed_dataset:476] [PID:2300] Unable to find prepared dataset in last_run_prepared/4818194e1a24026799335cd73096afba |
| [2026-04-05 19:19:12,380] [INFO] [axolotl.utils.data.sft._load_raw_datasets:320] [PID:2300] Loading raw datasets... |
| [2026-04-05 19:19:12,381] [WARNING] [axolotl.utils.data.sft._load_raw_datasets:322] [PID:2300] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. |
| [2026-04-05 19:19:14,389] [INFO] [axolotl.utils.data.wrappers.get_dataset_wrapper:87] [PID:2300] Loading dataset: mx003/cve with base_type: chat_template and prompt_style: None |
| [2026-04-05 19:19:14,411] [INFO] [axolotl.prompt_strategies.chat_template.__call__:969] [PID:2300] Using chat template: |
| --- |
| {{- bos_token }} |
| {%- if custom_tools is defined %} |
| {%- set tools = custom_tools %} |
| {%- endif %} |
| {%- if not tools_in_user_message is defined %} |
| {%- set tools_in_user_message = true %} |
| {%- endif %} |
| {%- if not date_string is defined %} |
| {%- set date_string = "26 Jul 2024" %} |
| {%- endif %} |
| {%- if not tools is defined %} |
| {%- set tools = none %} |
| {%- endif %} |
|
|
| {#- This block extracts the system message, so we can slot it into the right place. #} |
| {%- if messages[0]['role'] == 'system' %} |
| {%- set system_message = messages[0]['content']|trim %} |
| {%- set messages = messages[1:] %} |
| {%- else %} |
| {%- set system_message = "" %} |
| {%- endif %} |
|
|
| {#- System message + builtin tools #} |
| {{- "<|start_header_id|>system<|end_header_id|>\n\n" }} |
| {%- if builtin_tools is defined or tools is not none %} |
| {{- "Environment: ipython\n" }} |
| {%- endif %} |
| {%- if builtin_tools is defined %} |
| {{- "Tools: " + builtin_tools | reject('equalto', 'code_interpreter') | join(", ") + "\n\n"}} |
| {%- endif %} |
| {{- "Cutting Knowledge Date: December 2023\n" }} |
| {{- "Today Date: " + date_string + "\n\n" }} |
| {%- if tools is not none and not tools_in_user_message %} |
| {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }} |
| {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }} |
| {{- "Do not use variables.\n\n" }} |
| {%- for t in tools %} |
| {{- t | tojson(indent=4) }} |
| {{- "\n\n" }} |
| {%- endfor %} |
| {%- endif %} |
| {{- system_message }} |
| {{- "<|eot_id|>" }} |
|
|
| {#- Custom tools are passed in a user message with some extra guidance #} |
| {%- if tools_in_user_message and not tools is none %} |
| {#- Extract the first user message so we can plug it in here #} |
| {%- if messages | length != 0 %} |
| {%- set first_user_message = messages[0]['content']|trim %} |
| {%- set messages = messages[1:] %} |
| {%- else %} |
| {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }} |
| {%- endif %} |
| {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}} |
| {{- "Given the following functions, please respond with a JSON for a function call " }} |
| {{- "with its proper arguments that best answers the given prompt.\n\n" }} |
| {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }} |
| {{- "Do not use variables.\n\n" }} |
| {%- for t in tools %} |
| {{- t | tojson(indent=4) }} |
| {{- "\n\n" }} |
| {%- endfor %} |
| {{- first_user_message + "<|eot_id|>"}} |
| {%- endif %} |
|
|
| {%- for message in messages %} |
| {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %} |
| {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }} |
| {%- elif 'tool_calls' in message %} |
| {%- if not message.tool_calls|length == 1 %} |
| {{- raise_exception("This model only supports single tool-calls at once!") }} |
| {%- endif %} |
| {%- set tool_call = message.tool_calls[0].function %} |
| {%- if builtin_tools is defined and tool_call.name in builtin_tools %} |
| {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}} |
| {{- "<|python_tag|>" + tool_call.name + ".call(" }} |
| {%- for arg_name, arg_val in tool_call.arguments | items %} |
| {{- arg_name + '="' + arg_val + '"' }} |
| {%- if not loop.last %} |
| {{- ", " }} |
| {%- endif %} |
| {%- endfor %} |
| {{- ")" }} |
| {%- else %} |
| {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}} |
| {{- '{"name": "' + tool_call.name + '", ' }} |
| {{- '"parameters": ' }} |
| {{- tool_call.arguments | tojson }} |
| {{- "}" }} |
| {%- endif %} |
| {%- if builtin_tools is defined %} |
| {#- This means we're in ipython mode #} |
| {{- "<|eom_id|>" }} |
| {%- else %} |
| {{- "<|eot_id|>" }} |
| {%- endif %} |
| {%- elif message.role == "tool" or message.role == "ipython" %} |
| {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }} |
| {%- if message.content is mapping or message.content is iterable %} |
| {{- message.content | tojson }} |
| {%- else %} |
| {{- message.content }} |
| {%- endif %} |
| {{- "<|eot_id|>" }} |
| {%- endif %} |
| {%- endfor %} |
| {%- if add_generation_prompt %} |
| {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }} |
| {%- endif %} |
|
|
| --- |
|
Tokenizing Prompts (num_proc=26): 0%| | 0/485 [00:00<?, ? examples/s]
Tokenizing Prompts (num_proc=26): 4%|██▋ | 19/485 [00:01<00:31, 14.69 examples/s]
Tokenizing Prompts (num_proc=26): 8%|█████▎ | 38/485 [00:01<00:14, 29.91 examples/s]
Tokenizing Prompts (num_proc=26): 20%|█████████████▎ | 95/485 [00:01<00:04, 85.06 examples/s]
Tokenizing Prompts (num_proc=26): 24%|███████████████▋ | 114/485 [00:01<00:04, 90.67 examples/s]
Tokenizing Prompts (num_proc=26): 27%|██████████████████▎ | 133/485 [00:02<00:04, 78.49 examples/s]
Tokenizing Prompts (num_proc=26): 31%|████████████████████▉ | 152/485 [00:02<00:03, 87.21 examples/s]
Tokenizing Prompts (num_proc=26): 39%|██████████████████████████▏ | 190/485 [00:02<00:02, 99.01 examples/s]
Tokenizing Prompts (num_proc=26): 43%|████████████████████████████▍ | 209/485 [00:02<00:02, 111.29 examples/s]
Tokenizing Prompts (num_proc=26): 47%|███████████████████████████████▍ | 228/485 [00:03<00:03, 85.22 examples/s]
Tokenizing Prompts (num_proc=26): 51%|██████████████████████████████████ | 247/485 [00:03<00:02, 92.11 examples/s]
Tokenizing Prompts (num_proc=26): 55%|████████████████████████████████████▏ | 266/485 [00:03<00:02, 101.64 examples/s]
Tokenizing Prompts (num_proc=26): 59%|██████████████████████████████████████▊ | 285/485 [00:03<00:01, 112.55 examples/s]
Tokenizing Prompts (num_proc=26): 63%|█████████████████████████████████████████▎ | 304/485 [00:03<00:01, 121.42 examples/s]
Tokenizing Prompts (num_proc=26): 67%|███████████████████████████████████████████▉ | 323/485 [00:03<00:01, 124.05 examples/s]
Tokenizing Prompts (num_proc=26): 70%|██████████████████████████████████████████████▍ | 341/485 [00:03<00:01, 126.06 examples/s]
Tokenizing Prompts (num_proc=26): 74%|████████████████████████████████████████████████▊ | 359/485 [00:04<00:01, 122.11 examples/s]
Tokenizing Prompts (num_proc=26): 78%|███████████████████████████████████████████████████▎ | 377/485 [00:04<00:00, 119.17 examples/s]
Tokenizing Prompts (num_proc=26): 81%|█████████████████████████████████████████████████████▊ | 395/485 [00:04<00:00, 120.93 examples/s]
Tokenizing Prompts (num_proc=26): 89%|██████████████████████████████████████████████████████████▋ | 431/485 [00:04<00:00, 132.25 examples/s]
Tokenizing Prompts (num_proc=26): 93%|██████████████████████████████████████████████████████████████ | 449/485 [00:04<00:00, 99.80 examples/s]
Tokenizing Prompts (num_proc=26): 100%|██████████████████████████████████████████████████████████████████| 485/485 [00:05<00:00, 127.14 examples/s]
Tokenizing Prompts (num_proc=26): 100%|███████████████████████████████████████████████████████████████████| 485/485 [00:05<00:00, 90.28 examples/s] |
| [2026-04-05 19:19:20,047] [INFO] [axolotl.utils.data.utils.handle_long_seq_in_dataset:218] [PID:2300] min_input_len: 502 |
| [2026-04-05 19:19:20,048] [INFO] [axolotl.utils.data.utils.handle_long_seq_in_dataset:220] [PID:2300] max_input_len: 8182 |
|
Dropping Long Sequences (>4096) (num_proc=26): 0%| | 0/485 [00:00<?, ? examples/s]
Dropping Long Sequences (>4096) (num_proc=26): 4%|██▏ | 19/485 [00:00<00:09, 47.99 examples/s]
Dropping Long Sequences (>4096) (num_proc=26): 100%|█████████████████████████████████████████████████████| 485/485 [00:00<00:00, 765.61 examples/s] |
| [2026-04-05 19:19:20,790] [WARNING] [axolotl.utils.data.utils.handle_long_seq_in_dataset:260] [PID:2300] Dropped 26 samples from dataset |
|
Saving the dataset (0/1 shards): 0%| | 0/459 [00:00<?, ? examples/s]
Saving the dataset (1/1 shards): 100%|█████████████████████████████████████████████████████████████████| 459/459 [00:00<00:00, 12125.47 examples/s]
Saving the dataset (1/1 shards): 100%|█████████████████████████████████████████████████████████████████| 459/459 [00:00<00:00, 11705.11 examples/s] |
| [2026-04-05 19:19:20,957] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:404] [PID:2300] total_num_tokens: 697_096 |
| [2026-04-05 19:19:20,967] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:422] [PID:2300] `total_supervised_tokens: 175_355` |
| [2026-04-05 19:19:20,968] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:520] [PID:2300] total_num_steps: 87 |
| [2026-04-05 19:19:20,968] [INFO] [axolotl.utils.data.sft._prepare_standard_dataset:121] [PID:2300] Maximum number of steps set at 87 |
| [2026-04-05 19:19:21,005] [DEBUG] [axolotl.train.setup_model_and_tokenizer:65] [PID:2300] Loading tokenizer... unsloth/Meta-Llama-3.1-8B-Instruct |
| [2026-04-05 19:19:21,824] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:278] [PID:2300] EOS: 128009 / <|eot_id|> |
| [2026-04-05 19:19:21,825] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:279] [PID:2300] BOS: 128000 / <|begin_of_text|> |
| [2026-04-05 19:19:21,826] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:280] [PID:2300] PAD: 128004 / <|finetune_right_pad_id|> |
| [2026-04-05 19:19:21,826] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:281] [PID:2300] UNK: None / None |
| [2026-04-05 19:19:21,826] [DEBUG] [axolotl.train.setup_model_and_tokenizer:74] [PID:2300] Loading model |
| [2026-04-05 19:19:21,950] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_evaluation_loop:87] [PID:2300] Patched Trainer.evaluation_loop with nanmean loss calculation |
| [2026-04-05 19:19:21,952] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_maybe_log_save_evaluate:138] [PID:2300] Patched Trainer._maybe_log_save_evaluate with nanmean loss calculation |
|
model.safetensors.index.json: 0.00B [00:00, ?B/s]
model.safetensors.index.json: 23.9kB [00:00, 32.0MB/s] |
|
model-00001-of-00004.safetensors: 0%| | 0.00/4.98G [00:00<?, ?B/s]
model-00001-of-00004.safetensors: 0%| | 8.13k/4.98G [00:02<474:11:25, 2.92kB/s]
model-00001-of-00004.safetensors: 0%| | 2.23M/4.98G [00:03<1:47:41, 770kB/s]
model-00001-of-00004.safetensors: 0%| | 5.15M/4.98G [00:04<59:54, 1.38MB/s]
model-00001-of-00004.safetensors: 0%| | 5.41M/4.98G [00:05<1:01:01, 1.36MB/s]
model-00001-of-00004.safetensors: 0%|▏ | 13.7M/4.98G [00:05<18:50, 4.39MB/s]
model-00001-of-00004.safetensors: 0%|▏ | 16.2M/4.98G [00:06<17:03, 4.85MB/s]
model-00001-of-00004.safetensors: 1%|▍ | 28.4M/4.98G [00:06<06:56, 11.9MB/s]
model-00001-of-00004.safetensors: 1%|▋ | 48.2M/4.98G [00:06<03:04, 26.8MB/s]
model-00001-of-00004.safetensors: 1%|▊ | 60.2M/4.98G [00:06<03:11, 25.7MB/s]
model-00001-of-00004.safetensors: 1%|█ | 74.5M/4.98G [00:07<03:56, 20.7MB/s]
model-00001-of-00004.safetensors: 2%|█▍ | 96.5M/4.98G [00:08<03:00, 27.1MB/s]
model-00001-of-00004.safetensors: 2%|█▌ | 105M/4.98G [00:08<02:50, 28.5MB/s]
model-00001-of-00004.safetensors: 2%|█▌ | 112M/4.98G [00:09<04:21, 18.6MB/s]
model-00001-of-00004.safetensors: 2%|█▋ | 120M/4.98G [00:09<03:48, 21.3MB/s]
model-00001-of-00004.safetensors: 3%|█▉ | 135M/4.98G [00:10<02:45, 29.3MB/s]
model-00001-of-00004.safetensors: 3%|██ | 143M/4.98G [00:10<03:25, 23.5MB/s]
model-00001-of-00004.safetensors: 3%|██▎ | 163M/4.98G [00:11<03:20, 24.1MB/s]
model-00001-of-00004.safetensors: 3%|██▍ | 171M/4.98G [00:12<05:25, 14.8MB/s]
model-00001-of-00004.safetensors: 4%|██▌ | 180M/4.98G [00:13<04:56, 16.2MB/s]
model-00001-of-00004.safetensors: 4%|██▋ | 188M/4.98G [00:13<03:57, 20.2MB/s]
model-00001-of-00004.safetensors: 4%|██▉ | 200M/4.98G [00:13<03:05, 25.8MB/s]
model-00001-of-00004.safetensors: 4%|███▏ | 222M/4.98G [00:13<02:31, 31.5MB/s]
model-00001-of-00004.safetensors: 5%|███▊ | 262M/4.98G [00:14<01:15, 62.3MB/s]
model-00001-of-00004.safetensors: 6%|███▉ | 276M/4.98G [00:14<01:26, 54.4MB/s]
model-00001-of-00004.safetensors: 6%|████▏ | 289M/4.98G [00:16<04:11, 18.7MB/s]
model-00001-of-00004.safetensors: 6%|████▋ | 323M/4.98G [00:18<04:29, 17.3MB/s]
model-00001-of-00004.safetensors: 7%|████▊ | 336M/4.98G [00:19<04:10, 18.5MB/s]
model-00001-of-00004.safetensors: 7%|█████▎ | 369M/4.98G [00:19<02:41, 28.5MB/s]
model-00001-of-00004.safetensors: 8%|█████▍ | 380M/4.98G [00:19<02:30, 30.6MB/s]
model-00001-of-00004.safetensors: 8%|█████▋ | 392M/4.98G [00:20<03:00, 25.4MB/s]
model-00001-of-00004.safetensors: 8%|█████▉ | 413M/4.98G [00:20<02:20, 32.5MB/s]
model-00001-of-00004.safetensors: 9%|██████▎ | 435M/4.98G [00:21<02:33, 29.5MB/s]
model-00001-of-00004.safetensors: 9%|██████▋ | 460M/4.98G [00:22<01:48, 41.8MB/s]
model-00001-of-00004.safetensors: 10%|██████▉ | 477M/4.98G [00:22<02:20, 32.1MB/s]
model-00001-of-00004.safetensors: 10%|███████▍ | 511M/4.98G [00:26<04:30, 16.5MB/s]
model-00001-of-00004.safetensors: 11%|███████▊ | 544M/4.98G [00:27<03:52, 19.1MB/s]
model-00001-of-00004.safetensors: 12%|████████▌ | 590M/4.98G [00:27<02:22, 30.8MB/s]
model-00001-of-00004.safetensors: 13%|█████████▌ | 657M/4.98G [00:28<01:41, 42.4MB/s]
model-00001-of-00004.safetensors: 14%|██████████▎ | 710M/4.98G [00:29<01:37, 43.6MB/s]
model-00001-of-00004.safetensors: 16%|███████████▏ | 777M/4.98G [00:31<01:31, 45.7MB/s]
model-00001-of-00004.safetensors: 17%|████████████ | 836M/4.98G [00:31<01:04, 63.8MB/s]
model-00001-of-00004.safetensors: 18%|████████████▊ | 886M/4.98G [00:31<00:52, 78.2MB/s]
model-00001-of-00004.safetensors: 19%|█████████████▉ | 953M/4.98G [00:32<00:39, 101MB/s]
model-00001-of-00004.safetensors: 20%|██████████████▋ | 1.02G/4.98G [00:32<00:36, 108MB/s]
model-00001-of-00004.safetensors: 22%|███████████████▍ | 1.08G/4.98G [00:33<00:42, 91.6MB/s]
model-00001-of-00004.safetensors: 23%|████████████████▍ | 1.15G/4.98G [00:34<00:39, 95.8MB/s]
model-00001-of-00004.safetensors: 24%|█████████████████▋ | 1.22G/4.98G [00:34<00:33, 114MB/s]
model-00001-of-00004.safetensors: 26%|██████████████████▌ | 1.29G/4.98G [00:34<00:25, 146MB/s]
model-00001-of-00004.safetensors: 27%|███████████████████▌ | 1.35G/4.98G [00:34<00:22, 159MB/s]
model-00001-of-00004.safetensors: 29%|████████████████████▌ | 1.42G/4.98G [00:35<00:23, 150MB/s]
model-00001-of-00004.safetensors: 30%|█████████████████████▌ | 1.49G/4.98G [00:36<00:31, 111MB/s]
model-00001-of-00004.safetensors: 31%|█████████████████████▋ | 1.52G/4.98G [00:37<00:40, 85.8MB/s]
model-00001-of-00004.safetensors: 32%|███████████████████████ | 1.59G/4.98G [00:37<00:30, 111MB/s]
model-00001-of-00004.safetensors: 33%|███████████████████████▉ | 1.66G/4.98G [00:37<00:23, 144MB/s]
model-00001-of-00004.safetensors: 34%|████████████████████████▌ | 1.69G/4.98G [00:38<00:24, 133MB/s]
model-00001-of-00004.safetensors: 35%|█████████████████████████▍ | 1.76G/4.98G [00:38<00:21, 152MB/s]
model-00001-of-00004.safetensors: 36%|█████████████████████████▊ | 1.81G/4.98G [00:39<00:38, 82.6MB/s]
model-00001-of-00004.safetensors: 37%|██████████████████████████▏ | 1.84G/4.98G [00:40<00:37, 83.0MB/s]
model-00001-of-00004.safetensors: 38%|███████████████████████████▌ | 1.90G/4.98G [00:40<00:29, 106MB/s]
model-00001-of-00004.safetensors: 40%|█████████████████████████████ | 2.01G/4.98G [00:40<00:18, 157MB/s]
model-00001-of-00004.safetensors: 41%|█████████████████████████████▋ | 2.05G/4.98G [00:40<00:16, 182MB/s]
model-00001-of-00004.safetensors: 43%|██████████████████████████████▋ | 2.12G/4.98G [00:41<00:21, 135MB/s]
model-00001-of-00004.safetensors: 44%|███████████████████████████████▋ | 2.19G/4.98G [00:41<00:19, 145MB/s]
model-00001-of-00004.safetensors: 45%|████████████████████████████████▋ | 2.26G/4.98G [00:42<00:16, 164MB/s]
model-00001-of-00004.safetensors: 47%|█████████████████████████████████▌ | 2.32G/4.98G [00:42<00:15, 171MB/s]
model-00001-of-00004.safetensors: 48%|██████████████████████████████████ | 2.39G/4.98G [00:44<00:28, 91.0MB/s]
model-00001-of-00004.safetensors: 49%|██████████████████████████████████▌ | 2.42G/4.98G [00:44<00:28, 89.2MB/s]
model-00001-of-00004.safetensors: 50%|███████████████████████████████████▌ | 2.49G/4.98G [00:45<00:30, 81.9MB/s]
model-00001-of-00004.safetensors: 51%|████████████████████████████████████▍ | 2.56G/4.98G [00:46<00:32, 75.5MB/s]
model-00001-of-00004.safetensors: 53%|█████████████████████████████████████▍ | 2.63G/4.98G [00:47<00:37, 62.2MB/s]
model-00001-of-00004.safetensors: 54%|██████████████████████████████████████▍ | 2.69G/4.98G [00:48<00:27, 83.2MB/s]
model-00001-of-00004.safetensors: 55%|███████████████████████████████████████▎ | 2.76G/4.98G [00:48<00:22, 97.9MB/s]
model-00001-of-00004.safetensors: 57%|████████████████████████████████████████▉ | 2.83G/4.98G [00:49<00:19, 110MB/s]
model-00001-of-00004.safetensors: 58%|█████████████████████████████████████████▊ | 2.89G/4.98G [00:49<00:16, 127MB/s]
model-00001-of-00004.safetensors: 59%|██████████████████████████████████████████▊ | 2.96G/4.98G [00:49<00:12, 162MB/s]
model-00001-of-00004.safetensors: 61%|███████████████████████████████████████████▏ | 3.03G/4.98G [00:51<00:29, 67.1MB/s]
model-00001-of-00004.safetensors: 62%|████████████████████████████████████████████▏ | 3.09G/4.98G [00:52<00:25, 73.8MB/s]
model-00001-of-00004.safetensors: 64%|█████████████████████████████████████████████ | 3.16G/4.98G [00:54<00:35, 50.5MB/s]
model-00001-of-00004.safetensors: 65%|██████████████████████████████████████████████ | 3.23G/4.98G [00:55<00:26, 65.5MB/s]
model-00001-of-00004.safetensors: 66%|██████████████████████████████████████████████▋ | 3.27G/4.98G [00:55<00:25, 67.7MB/s]
model-00001-of-00004.safetensors: 67%|███████████████████████████████████████████████▌ | 3.34G/4.98G [00:59<00:44, 36.6MB/s]
model-00001-of-00004.safetensors: 68%|████████████████████████████████████████████████▌ | 3.40G/4.98G [01:02<00:51, 30.4MB/s]
model-00001-of-00004.safetensors: 70%|█████████████████████████████████████████████████▌ | 3.47G/4.98G [01:03<00:44, 33.8MB/s]
model-00001-of-00004.safetensors: 71%|██████████████████████████████████████████████████▍ | 3.54G/4.98G [01:05<00:39, 36.4MB/s]
model-00001-of-00004.safetensors: 72%|███████████████████████████████████████████████████▍ | 3.61G/4.98G [01:06<00:34, 40.0MB/s]
model-00001-of-00004.safetensors: 74%|████████████████████████████████████████████████████▎ | 3.66G/4.98G [01:08<00:37, 35.3MB/s]
model-00001-of-00004.safetensors: 75%|█████████████████████████████████████████████████████▏ | 3.73G/4.98G [01:08<00:25, 49.3MB/s]
model-00001-of-00004.safetensors: 76%|██████████████████████████████████████████████████████▏ | 3.80G/4.98G [01:10<00:22, 53.0MB/s]
model-00001-of-00004.safetensors: 78%|███████████████████████████████████████████████████████▏ | 3.87G/4.98G [01:10<00:16, 67.3MB/s]
model-00001-of-00004.safetensors: 79%|████████████████████████████████████████████████████████ | 3.93G/4.98G [01:11<00:17, 58.0MB/s]
model-00001-of-00004.safetensors: 80%|█████████████████████████████████████████████████████████ | 4.00G/4.98G [01:13<00:16, 58.9MB/s]
model-00001-of-00004.safetensors: 82%|██████████████████████████████████████████████████████████ | 4.07G/4.98G [01:13<00:12, 71.3MB/s]
model-00001-of-00004.safetensors: 83%|██████████████████████████████████████████████████████████▋ | 4.11G/4.98G [01:13<00:10, 80.9MB/s]
model-00001-of-00004.safetensors: 84%|████████████████████████████████████████████████████████████▍ | 4.18G/4.98G [01:13<00:07, 110MB/s]
model-00001-of-00004.safetensors: 85%|████████████████████████████████████████████████████████████▍ | 4.24G/4.98G [01:16<00:12, 60.2MB/s]
model-00001-of-00004.safetensors: 87%|█████████████████████████████████████████████████████████████▍ | 4.31G/4.98G [01:16<00:09, 69.0MB/s]
model-00001-of-00004.safetensors: 88%|██████████████████████████████████████████████████████████████▍ | 4.37G/4.98G [01:17<00:07, 84.7MB/s]
model-00001-of-00004.safetensors: 89%|███████████████████████████████████████████████████████████████▎ | 4.44G/4.98G [01:17<00:05, 95.9MB/s]
model-00001-of-00004.safetensors: 91%|█████████████████████████████████████████████████████████████████▏ | 4.51G/4.98G [01:17<00:03, 124MB/s]
model-00001-of-00004.safetensors: 92%|██████████████████████████████████████████████████████████████████▏ | 4.57G/4.98G [01:18<00:02, 151MB/s]
model-00001-of-00004.safetensors: 93%|██████████████████████████████████████████████████████████████████▏ | 4.64G/4.98G [01:21<00:06, 49.2MB/s]
model-00001-of-00004.safetensors: 95%|███████████████████████████████████████████████████████████████████▏ | 4.71G/4.98G [01:23<00:05, 44.7MB/s]
model-00001-of-00004.safetensors: 96%|████████████████████████████████████████████████████████████████████▏ | 4.78G/4.98G [01:28<00:07, 26.0MB/s]
model-00001-of-00004.safetensors: 97%|█████████████████████████████████████████████████████████████████████ | 4.84G/4.98G [01:31<00:05, 24.6MB/s]
model-00001-of-00004.safetensors: 99%|██████████████████████████████████████████████████████████████████████ | 4.91G/4.98G [01:32<00:02, 29.2MB/s]
model-00001-of-00004.safetensors: 99%|██████████████████████████████████████████████████████████████████████ | 4.91G/4.98G [01:44<00:02, 29.2MB/s]
model-00001-of-00004.safetensors: 100%|███████████████████████████████████████████████████████████████████████| 4.98G/4.98G [01:45<00:00, 12.4MB/s]
model-00001-of-00004.safetensors: 100%|███████████████████████████████████████████████████████████████████████| 4.98G/4.98G [01:45<00:00, 47.2MB/s] |
|
model-00002-of-00004.safetensors: 0%| | 0.00/5.00G [00:00<?, ?B/s]
model-00002-of-00004.safetensors: 0%| | 8.13k/5.00G [00:04<854:04:44, 1.63kB/s]
model-00002-of-00004.safetensors: 0%| | 5.45M/5.00G [00:06<1:20:30, 1.03MB/s]
model-00002-of-00004.safetensors: 1%|▍ | 28.4M/5.00G [00:07<14:53, 5.56MB/s]
model-00002-of-00004.safetensors: 1%|▌ | 36.7M/5.00G [00:08<12:03, 6.86MB/s]
model-00002-of-00004.safetensors: 1%|▌ | 43.6M/5.00G [00:08<09:28, 8.72MB/s]
model-00002-of-00004.safetensors: 1%|▉ | 63.9M/5.00G [00:09<06:20, 13.0MB/s]
model-00002-of-00004.safetensors: 1%|█ | 72.6M/5.00G [00:09<05:16, 15.6MB/s]
model-00002-of-00004.safetensors: 2%|█▏ | 80.7M/5.00G [00:10<04:49, 17.0MB/s]
model-00002-of-00004.safetensors: 2%|█▎ | 91.3M/5.00G [00:10<04:22, 18.7MB/s]
model-00002-of-00004.safetensors: 2%|█▋ | 114M/5.00G [00:10<02:27, 33.1MB/s]
model-00002-of-00004.safetensors: 2%|█▊ | 122M/5.00G [00:11<03:04, 26.5MB/s]
model-00002-of-00004.safetensors: 3%|██ | 140M/5.00G [00:11<02:14, 36.1MB/s]
model-00002-of-00004.safetensors: 3%|██▏ | 149M/5.00G [00:11<02:37, 30.7MB/s]
model-00002-of-00004.safetensors: 3%|██▎ | 157M/5.00G [00:12<02:34, 31.4MB/s]
model-00002-of-00004.safetensors: 3%|██▍ | 165M/5.00G [00:12<03:26, 23.4MB/s]
model-00002-of-00004.safetensors: 3%|██▌ | 174M/5.00G [00:13<03:25, 23.5MB/s]
model-00002-of-00004.safetensors: 4%|██▊ | 194M/5.00G [00:13<03:07, 25.6MB/s]
model-00002-of-00004.safetensors: 4%|██▉ | 203M/5.00G [00:14<03:23, 23.6MB/s]
model-00002-of-00004.safetensors: 4%|███▏ | 218M/5.00G [00:14<03:05, 25.7MB/s]
model-00002-of-00004.safetensors: 5%|███▎ | 226M/5.00G [00:14<02:48, 28.4MB/s]
model-00002-of-00004.safetensors: 5%|███▍ | 242M/5.00G [00:15<02:16, 35.0MB/s]
model-00002-of-00004.safetensors: 5%|███▌ | 251M/5.00G [00:15<02:05, 37.8MB/s]
model-00002-of-00004.safetensors: 5%|███▊ | 266M/5.00G [00:15<01:56, 40.6MB/s]
model-00002-of-00004.safetensors: 6%|████ | 284M/5.00G [00:16<02:59, 26.2MB/s]
model-00002-of-00004.safetensors: 6%|████▎ | 301M/5.00G [00:17<03:25, 22.8MB/s]
model-00002-of-00004.safetensors: 6%|████▋ | 325M/5.00G [00:18<02:31, 30.9MB/s]
model-00002-of-00004.safetensors: 7%|████▊ | 333M/5.00G [00:18<02:50, 27.3MB/s]
model-00002-of-00004.safetensors: 7%|████▉ | 341M/5.00G [00:18<02:32, 30.5MB/s]
model-00002-of-00004.safetensors: 7%|█████▍ | 375M/5.00G [00:18<01:23, 55.5MB/s]
model-00002-of-00004.safetensors: 8%|█████▋ | 392M/5.00G [00:19<01:39, 46.1MB/s]
model-00002-of-00004.safetensors: 8%|█████▊ | 401M/5.00G [00:22<05:18, 14.4MB/s]
model-00002-of-00004.safetensors: 8%|█████▉ | 411M/5.00G [00:22<04:31, 16.9MB/s]
model-00002-of-00004.safetensors: 9%|██████▊ | 474M/5.00G [00:23<02:18, 32.6MB/s]
model-00002-of-00004.safetensors: 10%|███████ | 488M/5.00G [00:23<02:09, 34.9MB/s]
model-00002-of-00004.safetensors: 11%|████████▏ | 565M/5.00G [00:25<02:02, 36.3MB/s]
model-00002-of-00004.safetensors: 12%|████████▍ | 584M/5.00G [00:26<01:56, 37.8MB/s]
model-00002-of-00004.safetensors: 13%|█████████ | 627M/5.00G [00:26<01:33, 46.9MB/s]
model-00002-of-00004.safetensors: 13%|█████████▏ | 641M/5.00G [00:29<03:08, 23.1MB/s]
model-00002-of-00004.safetensors: 14%|██████████▏ | 708M/5.00G [00:29<01:45, 40.6MB/s]
model-00002-of-00004.safetensors: 15%|██████████▌ | 731M/5.00G [00:30<01:53, 37.7MB/s]
model-00002-of-00004.safetensors: 16%|███████████▍ | 795M/5.00G [00:30<01:14, 56.4MB/s]
model-00002-of-00004.safetensors: 17%|████████████▍ | 862M/5.00G [00:30<00:52, 78.4MB/s]
model-00002-of-00004.safetensors: 18%|████████████▉ | 903M/5.00G [00:31<00:50, 80.7MB/s]
model-00002-of-00004.safetensors: 19%|█████████████▌ | 944M/5.00G [00:31<00:43, 93.3MB/s]
model-00002-of-00004.safetensors: 19%|█████████████▊ | 961M/5.00G [00:33<01:29, 45.3MB/s]
model-00002-of-00004.safetensors: 21%|██████████████▌ | 1.03G/5.00G [00:34<01:12, 54.5MB/s]
model-00002-of-00004.safetensors: 22%|███████████████▌ | 1.10G/5.00G [00:34<00:51, 75.8MB/s]
model-00002-of-00004.safetensors: 23%|████████████████ | 1.13G/5.00G [00:34<00:45, 84.5MB/s]
model-00002-of-00004.safetensors: 23%|████████████████▍ | 1.16G/5.00G [00:34<00:40, 95.3MB/s]
model-00002-of-00004.safetensors: 24%|█████████████████▍ | 1.21G/5.00G [00:35<00:30, 124MB/s]
model-00002-of-00004.safetensors: 26%|██████████████████▎ | 1.27G/5.00G [00:35<00:27, 133MB/s]
model-00002-of-00004.safetensors: 26%|██████████████████▋ | 1.30G/5.00G [00:35<00:33, 110MB/s]
model-00002-of-00004.safetensors: 27%|███████████████████▋ | 1.37G/5.00G [00:36<00:24, 150MB/s]
model-00002-of-00004.safetensors: 29%|████████████████████▋ | 1.43G/5.00G [00:36<00:21, 169MB/s]
model-00002-of-00004.safetensors: 30%|█████████████████████▌ | 1.50G/5.00G [00:36<00:17, 198MB/s]
model-00002-of-00004.safetensors: 31%|██████████████████████▌ | 1.57G/5.00G [00:37<00:17, 196MB/s]
model-00002-of-00004.safetensors: 32%|███████████████████████▏ | 1.61G/5.00G [00:37<00:21, 160MB/s]
model-00002-of-00004.safetensors: 34%|████████████████████████▏ | 1.68G/5.00G [00:37<00:16, 196MB/s]
model-00002-of-00004.safetensors: 35%|█████████████████████████▏ | 1.75G/5.00G [00:37<00:14, 220MB/s]
model-00002-of-00004.safetensors: 36%|██████████████████████████ | 1.81G/5.00G [00:38<00:16, 191MB/s]
model-00002-of-00004.safetensors: 38%|███████████████████████████ | 1.88G/5.00G [00:38<00:13, 224MB/s]
model-00002-of-00004.safetensors: 39%|████████████████████████████▏ | 1.95G/5.00G [00:38<00:14, 216MB/s]
model-00002-of-00004.safetensors: 40%|█████████████████████████████ | 2.02G/5.00G [00:39<00:14, 207MB/s]
model-00002-of-00004.safetensors: 41%|█████████████████████████████▌ | 2.06G/5.00G [00:39<00:14, 200MB/s]
model-00002-of-00004.safetensors: 42%|██████████████████████████████▌ | 2.12G/5.00G [00:39<00:11, 241MB/s]
model-00002-of-00004.safetensors: 44%|███████████████████████████████▌ | 2.19G/5.00G [00:39<00:10, 271MB/s]
model-00002-of-00004.safetensors: 45%|████████████████████████████████▍ | 2.26G/5.00G [00:39<00:08, 316MB/s]
model-00002-of-00004.safetensors: 46%|█████████████████████████████████▍ | 2.32G/5.00G [00:40<00:07, 357MB/s]
model-00002-of-00004.safetensors: 48%|██████████████████████████████████▍ | 2.39G/5.00G [00:40<00:07, 359MB/s]
model-00002-of-00004.safetensors: 49%|███████████████████████████████████▍ | 2.46G/5.00G [00:40<00:07, 334MB/s]
model-00002-of-00004.safetensors: 50%|████████████████████████████████████▎ | 2.52G/5.00G [00:40<00:08, 300MB/s]
model-00002-of-00004.safetensors: 51%|████████████████████████████████████▉ | 2.57G/5.00G [00:40<00:07, 309MB/s]
model-00002-of-00004.safetensors: 52%|█████████████████████████████████████▍ | 2.60G/5.00G [00:40<00:07, 309MB/s]
model-00002-of-00004.safetensors: 53%|██████████████████████████████████████▍ | 2.67G/5.00G [00:41<00:10, 232MB/s]
model-00002-of-00004.safetensors: 54%|███████████████████████████████████████ | 2.71G/5.00G [00:41<00:10, 225MB/s]
model-00002-of-00004.safetensors: 56%|███████████████████████████████████████▉ | 2.78G/5.00G [00:41<00:08, 257MB/s]
model-00002-of-00004.safetensors: 57%|████████████████████████████████████████▉ | 2.84G/5.00G [00:41<00:07, 304MB/s]
model-00002-of-00004.safetensors: 58%|█████████████████████████████████████████▌ | 2.89G/5.00G [00:42<00:06, 308MB/s]
model-00002-of-00004.safetensors: 59%|██████████████████████████████████████████▌ | 2.95G/5.00G [00:42<00:09, 210MB/s]
model-00002-of-00004.safetensors: 61%|███████████████████████████████████████████▋ | 3.04G/5.00G [00:42<00:07, 261MB/s]
model-00002-of-00004.safetensors: 62%|████████████████████████████████████████████▋ | 3.10G/5.00G [00:42<00:06, 308MB/s]
model-00002-of-00004.safetensors: 63%|█████████████████████████████████████████████▎ | 3.15G/5.00G [00:43<00:05, 320MB/s]
model-00002-of-00004.safetensors: 65%|██████████████████████████████████████████████▋ | 3.25G/5.00G [00:43<00:07, 242MB/s]
model-00002-of-00004.safetensors: 66%|███████████████████████████████████████████████▎ | 3.29G/5.00G [00:43<00:07, 235MB/s]
model-00002-of-00004.safetensors: 67%|████████████████████████████████████████████████▎ | 3.36G/5.00G [00:44<00:08, 188MB/s]
model-00002-of-00004.safetensors: 69%|█████████████████████████████████████████████████▉ | 3.47G/5.00G [00:44<00:06, 230MB/s]
model-00002-of-00004.safetensors: 71%|██████████████████████████████████████████████████▉ | 3.53G/5.00G [00:44<00:05, 267MB/s]
model-00002-of-00004.safetensors: 72%|███████████████████████████████████████████████████▏ | 3.60G/5.00G [00:47<00:16, 83.6MB/s]
model-00002-of-00004.safetensors: 73%|████████████████████████████████████████████████████ | 3.67G/5.00G [00:49<00:26, 51.1MB/s]
model-00002-of-00004.safetensors: 75%|█████████████████████████████████████████████████████ | 3.73G/5.00G [00:49<00:19, 64.9MB/s]
model-00002-of-00004.safetensors: 76%|█████████████████████████████████████████████████████▉ | 3.80G/5.00G [00:51<00:19, 60.7MB/s]
model-00002-of-00004.safetensors: 77%|██████████████████████████████████████████████████████▉ | 3.87G/5.00G [00:52<00:20, 55.2MB/s]
model-00002-of-00004.safetensors: 79%|███████████████████████████████████████████████████████▉ | 3.94G/5.00G [00:52<00:14, 74.6MB/s]
model-00002-of-00004.safetensors: 80%|████████████████████████████████████████████████████████▊ | 4.00G/5.00G [00:55<00:19, 50.5MB/s]
model-00002-of-00004.safetensors: 81%|█████████████████████████████████████████████████████████▊ | 4.07G/5.00G [00:55<00:14, 63.8MB/s]
model-00002-of-00004.safetensors: 83%|██████████████████████████████████████████████████████████▊ | 4.14G/5.00G [00:55<00:10, 84.9MB/s]
model-00002-of-00004.safetensors: 84%|████████████████████████████████████████████████████████████▌ | 4.20G/5.00G [00:55<00:07, 110MB/s]
model-00002-of-00004.safetensors: 85%|████████████████████████████████████████████████████████████▋ | 4.27G/5.00G [00:57<00:09, 77.9MB/s]
model-00002-of-00004.safetensors: 87%|█████████████████████████████████████████████████████████████▌ | 4.34G/5.00G [00:59<00:13, 50.8MB/s]
model-00002-of-00004.safetensors: 88%|██████████████████████████████████████████████████████████████▌ | 4.41G/5.00G [01:00<00:09, 61.8MB/s]
model-00002-of-00004.safetensors: 89%|███████████████████████████████████████████████████████████████▌ | 4.47G/5.00G [01:00<00:07, 75.0MB/s]
model-00002-of-00004.safetensors: 91%|████████████████████████████████████████████████████████████████▍ | 4.54G/5.00G [01:01<00:06, 74.0MB/s]
model-00002-of-00004.safetensors: 92%|█████████████████████████████████████████████████████████████████▎ | 4.60G/5.00G [01:02<00:04, 87.1MB/s]
model-00002-of-00004.safetensors: 93%|██████████████████████████████████████████████████████████████████▏ | 4.66G/5.00G [01:04<00:05, 58.7MB/s]
model-00002-of-00004.safetensors: 95%|███████████████████████████████████████████████████████████████████▏ | 4.73G/5.00G [01:04<00:03, 70.0MB/s]
model-00002-of-00004.safetensors: 96%|████████████████████████████████████████████████████████████████████▏ | 4.80G/5.00G [01:04<00:02, 94.8MB/s]
model-00002-of-00004.safetensors: 97%|█████████████████████████████████████████████████████████████████████ | 4.87G/5.00G [01:06<00:02, 65.6MB/s]
model-00002-of-00004.safetensors: 99%|██████████████████████████████████████████████████████████████████████ | 4.93G/5.00G [01:07<00:01, 61.1MB/s]
model-00002-of-00004.safetensors: 100%|███████████████████████████████████████████████████████████████████████| 5.00G/5.00G [01:09<00:00, 50.0MB/s]
model-00002-of-00004.safetensors: 100%|███████████████████████████████████████████████████████████████████████| 5.00G/5.00G [01:09<00:00, 71.8MB/s] |
|
model-00003-of-00004.safetensors: 0%| | 0.00/4.92G [00:00<?, ?B/s]
model-00003-of-00004.safetensors: 0%| | 11.2k/4.92G [00:02<286:09:32, 4.77kB/s]
model-00003-of-00004.safetensors: 0%| | 22.0k/4.92G [00:03<192:47:13, 7.08kB/s]
model-00003-of-00004.safetensors: 0%| | 8.26M/4.92G [00:05<36:23, 2.25MB/s]
model-00003-of-00004.safetensors: 0%|▎ | 23.6M/4.92G [00:05<11:29, 7.09MB/s]
model-00003-of-00004.safetensors: 1%|▋ | 47.9M/4.92G [00:05<05:03, 16.0MB/s]
model-00003-of-00004.safetensors: 1%|▊ | 54.7M/4.92G [00:06<04:38, 17.5MB/s]
model-00003-of-00004.safetensors: 1%|▉ | 62.4M/4.92G [00:06<04:14, 19.1MB/s]
model-00003-of-00004.safetensors: 1%|█ | 70.8M/4.92G [00:06<03:32, 22.8MB/s]
model-00003-of-00004.safetensors: 2%|█▏ | 79.0M/4.92G [00:07<05:53, 13.7MB/s]
model-00003-of-00004.safetensors: 2%|█▏ | 85.7M/4.92G [00:08<05:36, 14.3MB/s]
model-00003-of-00004.safetensors: 2%|█▎ | 90.7M/4.92G [00:09<07:00, 11.5MB/s]
model-00003-of-00004.safetensors: 2%|█▍ | 99.0M/4.92G [00:09<06:49, 11.8MB/s]
model-00003-of-00004.safetensors: 2%|█▋ | 113M/4.92G [00:10<04:49, 16.6MB/s]
model-00003-of-00004.safetensors: 3%|█▉ | 131M/4.92G [00:10<02:59, 26.7MB/s]
model-00003-of-00004.safetensors: 3%|██ | 143M/4.92G [00:11<03:46, 21.0MB/s]
model-00003-of-00004.safetensors: 3%|██▎ | 154M/4.92G [00:11<03:15, 24.4MB/s]
model-00003-of-00004.safetensors: 3%|██▍ | 163M/4.92G [00:11<03:26, 23.0MB/s]
model-00003-of-00004.safetensors: 4%|██▋ | 183M/4.92G [00:12<02:59, 26.3MB/s]
model-00003-of-00004.safetensors: 4%|██▊ | 191M/4.92G [00:12<02:54, 27.1MB/s]
model-00003-of-00004.safetensors: 4%|██▉ | 199M/4.92G [00:13<02:50, 27.7MB/s]
model-00003-of-00004.safetensors: 4%|███▏ | 214M/4.92G [00:14<04:14, 18.5MB/s]
model-00003-of-00004.safetensors: 5%|███▎ | 228M/4.92G [00:14<03:17, 23.7MB/s]
model-00003-of-00004.safetensors: 5%|███▋ | 253M/4.92G [00:15<02:30, 31.0MB/s]
model-00003-of-00004.safetensors: 6%|███▉ | 273M/4.92G [00:15<02:52, 27.0MB/s]
model-00003-of-00004.safetensors: 6%|████▎ | 290M/4.92G [00:16<02:28, 31.2MB/s]
model-00003-of-00004.safetensors: 6%|████▍ | 306M/4.92G [00:16<02:02, 37.6MB/s]
model-00003-of-00004.safetensors: 7%|████▋ | 324M/4.92G [00:16<01:44, 43.9MB/s]
model-00003-of-00004.safetensors: 7%|████▉ | 340M/4.92G [00:17<01:34, 48.3MB/s]
model-00003-of-00004.safetensors: 8%|█████▍ | 369M/4.92G [00:20<04:57, 15.3MB/s]
model-00003-of-00004.safetensors: 8%|█████▉ | 402M/4.92G [00:21<03:41, 20.3MB/s]
model-00003-of-00004.safetensors: 9%|██████▍ | 436M/4.92G [00:21<02:29, 30.0MB/s]
model-00003-of-00004.safetensors: 10%|██████▊ | 469M/4.92G [00:23<02:43, 27.2MB/s]
model-00003-of-00004.safetensors: 10%|███████▏ | 487M/4.92G [00:24<03:30, 21.1MB/s]
model-00003-of-00004.safetensors: 11%|███████▊ | 531M/4.92G [00:25<02:41, 27.2MB/s]
model-00003-of-00004.safetensors: 12%|████████▍ | 573M/4.92G [00:26<02:01, 35.7MB/s]
model-00003-of-00004.safetensors: 12%|████████▉ | 606M/4.92G [00:27<02:10, 33.1MB/s]
model-00003-of-00004.safetensors: 13%|█████████▏ | 625M/4.92G [00:28<02:49, 25.3MB/s]
model-00003-of-00004.safetensors: 13%|█████████▍ | 646M/4.92G [00:29<02:31, 28.2MB/s]
model-00003-of-00004.safetensors: 14%|█████████▊ | 666M/4.92G [00:30<02:24, 29.4MB/s]
model-00003-of-00004.safetensors: 15%|██████████▍ | 717M/4.92G [00:30<01:30, 46.6MB/s]
model-00003-of-00004.safetensors: 15%|███████████ | 759M/4.92G [00:30<01:07, 61.8MB/s]
model-00003-of-00004.safetensors: 16%|███████████▊ | 805M/4.92G [00:30<00:52, 78.9MB/s]
model-00003-of-00004.safetensors: 18%|████████████▉ | 872M/4.92G [00:31<00:35, 113MB/s]
model-00003-of-00004.safetensors: 19%|█████████████▌ | 912M/4.92G [00:31<00:28, 138MB/s]
model-00003-of-00004.safetensors: 19%|█████████████▉ | 955M/4.92G [00:33<01:10, 55.9MB/s]
model-00003-of-00004.safetensors: 20%|██████████████▍ | 988M/4.92G [00:34<01:26, 45.5MB/s]
model-00003-of-00004.safetensors: 21%|██████████████▉ | 1.03G/4.92G [00:34<01:01, 63.0MB/s]
model-00003-of-00004.safetensors: 22%|███████████████▊ | 1.09G/4.92G [00:35<01:12, 53.1MB/s]
model-00003-of-00004.safetensors: 24%|████████████████▋ | 1.16G/4.92G [00:36<00:47, 79.8MB/s]
model-00003-of-00004.safetensors: 25%|█████████████████▋ | 1.23G/4.92G [00:36<00:39, 93.6MB/s]
model-00003-of-00004.safetensors: 26%|██████████████████▋ | 1.29G/4.92G [00:37<00:41, 88.2MB/s]
model-00003-of-00004.safetensors: 27%|███████████████████▍ | 1.34G/4.92G [00:38<00:43, 82.0MB/s]
model-00003-of-00004.safetensors: 29%|████████████████████▎ | 1.41G/4.92G [00:38<00:35, 98.9MB/s]
model-00003-of-00004.safetensors: 30%|█████████████████████▋ | 1.48G/4.92G [00:38<00:31, 110MB/s]
model-00003-of-00004.safetensors: 31%|██████████████████████▏ | 1.52G/4.92G [00:39<00:26, 127MB/s]
model-00003-of-00004.safetensors: 32%|███████████████████████▏ | 1.58G/4.92G [00:39<00:28, 118MB/s]
model-00003-of-00004.safetensors: 33%|███████████████████████▊ | 1.63G/4.92G [00:39<00:24, 134MB/s]
model-00003-of-00004.safetensors: 35%|████████████████████████▊ | 1.70G/4.92G [00:40<00:19, 164MB/s]
model-00003-of-00004.safetensors: 35%|█████████████████████████▍ | 1.74G/4.92G [00:40<00:17, 187MB/s]
model-00003-of-00004.safetensors: 37%|██████████████████████████▍ | 1.80G/4.92G [00:40<00:18, 169MB/s]
model-00003-of-00004.safetensors: 38%|███████████████████████████▍ | 1.87G/4.92G [00:41<00:17, 169MB/s]
model-00003-of-00004.safetensors: 39%|███████████████████████████▉ | 1.90G/4.92G [00:41<00:18, 160MB/s]
model-00003-of-00004.safetensors: 40%|████████████████████████████▉ | 1.97G/4.92G [00:41<00:14, 204MB/s]
model-00003-of-00004.safetensors: 41%|█████████████████████████████▊ | 2.04G/4.92G [00:41<00:11, 243MB/s]
model-00003-of-00004.safetensors: 43%|██████████████████████████████▊ | 2.11G/4.92G [00:42<00:23, 121MB/s]
model-00003-of-00004.safetensors: 44%|███████████████████████████████▊ | 2.17G/4.92G [00:43<00:17, 153MB/s]
model-00003-of-00004.safetensors: 46%|████████████████████████████████▊ | 2.24G/4.92G [00:44<00:24, 110MB/s]
model-00003-of-00004.safetensors: 47%|█████████████████████████████████▊ | 2.31G/4.92G [00:44<00:23, 110MB/s]
model-00003-of-00004.safetensors: 48%|██████████████████████████████████▍ | 2.35G/4.92G [00:44<00:19, 131MB/s]
model-00003-of-00004.safetensors: 49%|███████████████████████████████████▎ | 2.41G/4.92G [00:45<00:18, 135MB/s]
model-00003-of-00004.safetensors: 50%|████████████████████████████████████▎ | 2.48G/4.92G [00:45<00:16, 151MB/s]
model-00003-of-00004.safetensors: 51%|████████████████████████████████████▉ | 2.52G/4.92G [00:45<00:15, 150MB/s]
model-00003-of-00004.safetensors: 53%|█████████████████████████████████████▉ | 2.59G/4.92G [00:46<00:19, 117MB/s]
model-00003-of-00004.safetensors: 54%|██████████████████████████████████████▎ | 2.65G/4.92G [00:47<00:23, 95.1MB/s]
model-00003-of-00004.safetensors: 55%|███████████████████████████████████████▊ | 2.72G/4.92G [00:47<00:17, 125MB/s]
model-00003-of-00004.safetensors: 57%|████████████████████████████████████████▎ | 2.79G/4.92G [00:48<00:21, 98.7MB/s]
model-00003-of-00004.safetensors: 58%|█████████████████████████████████████████▊ | 2.86G/4.92G [00:49<00:16, 123MB/s]
model-00003-of-00004.safetensors: 59%|██████████████████████████████████████████▊ | 2.92G/4.92G [00:49<00:12, 155MB/s]
model-00003-of-00004.safetensors: 61%|███████████████████████████████████████████▊ | 2.99G/4.92G [00:49<00:10, 176MB/s]
model-00003-of-00004.safetensors: 62%|████████████████████████████████████████████▌ | 3.04G/4.92G [00:49<00:09, 197MB/s]
model-00003-of-00004.safetensors: 63%|█████████████████████████████████████████████▌ | 3.11G/4.92G [00:50<00:12, 143MB/s]
model-00003-of-00004.safetensors: 65%|██████████████████████████████████████████████▌ | 3.18G/4.92G [00:50<00:09, 188MB/s]
model-00003-of-00004.safetensors: 66%|███████████████████████████████████████████████▌ | 3.24G/4.92G [00:50<00:07, 221MB/s]
model-00003-of-00004.safetensors: 67%|████████████████████████████████████████████████▍ | 3.31G/4.92G [00:51<00:09, 166MB/s]
model-00003-of-00004.safetensors: 69%|█████████████████████████████████████████████████▍ | 3.37G/4.92G [00:51<00:07, 203MB/s]
model-00003-of-00004.safetensors: 70%|██████████████████████████████████████████████████▍ | 3.44G/4.92G [00:51<00:07, 193MB/s]
model-00003-of-00004.safetensors: 71%|███████████████████████████████████████████████████▎ | 3.51G/4.92G [00:52<00:09, 142MB/s]
model-00003-of-00004.safetensors: 73%|████████████████████████████████████████████████████▎ | 3.57G/4.92G [00:52<00:09, 147MB/s]
model-00003-of-00004.safetensors: 74%|█████████████████████████████████████████████████████▎ | 3.64G/4.92G [00:53<00:07, 172MB/s]
model-00003-of-00004.safetensors: 75%|██████████████████████████████████████████████████████▎ | 3.71G/4.92G [00:53<00:08, 150MB/s]
model-00003-of-00004.safetensors: 77%|███████████████████████████████████████████████████████▎ | 3.78G/4.92G [00:54<00:07, 143MB/s]
model-00003-of-00004.safetensors: 78%|████████████████████████████████████████████████████████▎ | 3.84G/4.92G [00:54<00:06, 166MB/s]
model-00003-of-00004.safetensors: 80%|█████████████████████████████████████████████████████████▎ | 3.91G/4.92G [00:55<00:08, 122MB/s]
model-00003-of-00004.safetensors: 81%|██████████████████████████████████████████████████████████▏ | 3.98G/4.92G [00:55<00:07, 133MB/s]
model-00003-of-00004.safetensors: 82%|███████████████████████████████████████████████████████████▏ | 4.04G/4.92G [00:56<00:06, 142MB/s]
model-00003-of-00004.safetensors: 84%|████████████████████████████████████████████████████████████▏ | 4.11G/4.92G [00:56<00:05, 152MB/s]
model-00003-of-00004.safetensors: 85%|█████████████████████████████████████████████████████████████▏ | 4.18G/4.92G [00:57<00:05, 139MB/s]
model-00003-of-00004.safetensors: 86%|██████████████████████████████████████████████████████████████▏ | 4.25G/4.92G [00:57<00:04, 153MB/s]
model-00003-of-00004.safetensors: 88%|███████████████████████████████████████████████████████████████▏ | 4.31G/4.92G [00:57<00:03, 191MB/s]
model-00003-of-00004.safetensors: 89%|████████████████████████████████████████████████████████████████▏ | 4.38G/4.92G [00:58<00:04, 117MB/s]
model-00003-of-00004.safetensors: 90%|████████████████████████████████████████████████████████████████▏ | 4.45G/4.92G [01:00<00:06, 72.3MB/s]
model-00003-of-00004.safetensors: 92%|█████████████████████████████████████████████████████████████████▏ | 4.51G/4.92G [01:00<00:04, 90.5MB/s]
model-00003-of-00004.safetensors: 93%|███████████████████████████████████████████████████████████████████ | 4.58G/4.92G [01:01<00:03, 107MB/s]
model-00003-of-00004.safetensors: 95%|████████████████████████████████████████████████████████████████████ | 4.65G/4.92G [01:01<00:02, 125MB/s]
model-00003-of-00004.safetensors: 96%|████████████████████████████████████████████████████████████████████ | 4.71G/4.92G [01:03<00:02, 67.6MB/s]
model-00003-of-00004.safetensors: 97%|█████████████████████████████████████████████████████████████████████ | 4.78G/4.92G [01:05<00:02, 51.7MB/s]
model-00003-of-00004.safetensors: 99%|██████████████████████████████████████████████████████████████████████ | 4.85G/4.92G [01:10<00:02, 28.0MB/s]
model-00003-of-00004.safetensors: 100%|███████████████████████████████████████████████████████████████████████| 4.92G/4.92G [01:15<00:00, 21.5MB/s]
model-00003-of-00004.safetensors: 100%|███████████████████████████████████████████████████████████████████████| 4.92G/4.92G [01:15<00:00, 65.2MB/s] |
|
model-00004-of-00004.safetensors: 0%| | 0.00/1.17G [00:00<?, ?B/s]
model-00004-of-00004.safetensors: 0%| | 224k/1.17G [00:01<2:12:53, 146kB/s]
model-00004-of-00004.safetensors: 0%| | 336k/1.17G [00:05<6:35:46, 49.2kB/s]
model-00004-of-00004.safetensors: 1%|▊ | 13.4M/1.17G [00:07<07:19, 2.63MB/s]
model-00004-of-00004.safetensors: 2%|█▍ | 23.9M/1.17G [00:07<03:46, 5.06MB/s]
model-00004-of-00004.safetensors: 3%|██ | 33.9M/1.17G [00:08<02:52, 6.59MB/s]
model-00004-of-00004.safetensors: 5%|███▊ | 63.7M/1.17G [00:13<02:52, 6.40MB/s]
model-00004-of-00004.safetensors: 11%|████████ | 131M/1.17G [00:17<01:41, 10.3MB/s]
model-00004-of-00004.safetensors: 16%|███████████▍ | 185M/1.17G [00:19<01:06, 14.7MB/s]
model-00004-of-00004.safetensors: 22%|███████████████▌ | 252M/1.17G [00:19<00:38, 23.6MB/s]
model-00004-of-00004.safetensors: 27%|███████████████████▋ | 319M/1.17G [00:20<00:24, 34.8MB/s]
model-00004-of-00004.safetensors: 32%|███████████████████████ | 373M/1.17G [00:20<00:17, 46.2MB/s]
model-00004-of-00004.safetensors: 38%|███████████████████████████▏ | 440M/1.17G [00:21<00:14, 50.7MB/s]
model-00004-of-00004.safetensors: 43%|███████████████████████████████▎ | 507M/1.17G [00:23<00:13, 47.6MB/s]
model-00004-of-00004.safetensors: 49%|███████████████████████████████████▍ | 575M/1.17G [00:24<00:12, 48.9MB/s]
model-00004-of-00004.safetensors: 54%|██████████████████████████████████████▉ | 632M/1.17G [00:25<00:09, 54.9MB/s]
model-00004-of-00004.safetensors: 60%|███████████████████████████████████████████ | 699M/1.17G [00:25<00:06, 74.8MB/s]
model-00004-of-00004.safetensors: 66%|███████████████████████████████████████████████▏ | 766M/1.17G [00:26<00:04, 91.6MB/s]
model-00004-of-00004.safetensors: 71%|███████████████████████████████████████████████████▎ | 833M/1.17G [00:27<00:05, 63.2MB/s]
model-00004-of-00004.safetensors: 77%|███████████████████████████████████████████████████████▍ | 900M/1.17G [00:28<00:03, 78.4MB/s]
model-00004-of-00004.safetensors: 83%|███████████████████████████████████████████████████████████▌ | 967M/1.17G [00:31<00:04, 42.8MB/s]
model-00004-of-00004.safetensors: 89%|██████████████████████████████████████████████████████████████▊ | 1.03G/1.17G [00:32<00:02, 45.1MB/s]
model-00004-of-00004.safetensors: 94%|██████████████████████████████████████████████████████████████████▉ | 1.10G/1.17G [00:33<00:01, 57.9MB/s]
model-00004-of-00004.safetensors: 100%|███████████████████████████████████████████████████████████████████████| 1.17G/1.17G [00:37<00:00, 30.1MB/s]
model-00004-of-00004.safetensors: 100%|███████████████████████████████████████████████████████████████████████| 1.17G/1.17G [00:37<00:00, 30.9MB/s] |
|
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████| 4/4 [00:00<00:00, 47.00it/s] |
|
generation_config.json: 0%| | 0.00/239 [00:00<?, ?B/s]
generation_config.json: 100%|██████████████████████████████████████████████████████████████████████████████████████| 239/239 [00:00<00:00, 628kB/s] |
| [2026-04-05 19:24:15,254] [INFO] [axolotl.loaders.model._configure_embedding_dtypes:345] [PID:2300] Converting modules to torch.bfloat16 |
| [2026-04-05 19:24:16,536] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:2300] Memory usage after model load 0.000GB () |
| trainable params: 83,886,080 || all params: 8,114,147,328 || trainable%: 1.0338 |
| [2026-04-05 19:24:17,207] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:2300] after adapters 0.000GB () |
| [2026-04-05 19:24:26,689] [INFO] [axolotl.train.save_initial_configs:398] [PID:2300] Pre-saving adapter config to ./outputs/mymodel... |
| [2026-04-05 19:24:26,695] [INFO] [axolotl.train.save_initial_configs:402] [PID:2300] Pre-saving tokenizer to ./outputs/mymodel... |
| [2026-04-05 19:24:27,070] [INFO] [axolotl.train.save_initial_configs:407] [PID:2300] Pre-saving model config to ./outputs/mymodel... |
| [2026-04-05 19:24:27,078] [INFO] [axolotl.train.execute_training:196] [PID:2300] Starting trainer... |
|
0%| | 0/87 [00:00<?, ?it/s]Traceback (most recent call last): |
| File "<frozen runpy>", line 198, in _run_module_as_main |
| File "<frozen runpy>", line 88, in _run_code |
| File "/workspace/axolotl/src/axolotl/cli/train.py", line 121, in <module> |
| fire.Fire(do_cli) |
| File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/fire/core.py", line 135, in Fire |
| component_trace = _Fire(component, args, parsed_flag_args, context, name) |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ |
| File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/fire/core.py", line 468, in _Fire |
| component, remaining_args = _CallAndUpdateTrace( |
| ^^^^^^^^^^^^^^^^^^^^ |
| File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/fire/core.py", line 684, in _CallAndUpdateTrace |
| component = fn(*varargs, **kwargs) |
| ^^^^^^^^^^^^^^^^^^^^^^ |
| File "/workspace/axolotl/src/axolotl/cli/train.py", line 88, in do_cli |
| return do_train(parsed_cfg, parsed_cli_args) |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ |
| File "/workspace/axolotl/src/axolotl/cli/train.py", line 45, in do_train |
| model, tokenizer, trainer = train(cfg=cfg, dataset_meta=dataset_meta) |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ |
| File "/workspace/axolotl/src/axolotl/train.py", line 573, in train |
| execute_training(cfg, trainer, resume_from_checkpoint) |
| File "/workspace/axolotl/src/axolotl/train.py", line 197, in execute_training |
| trainer.train(resume_from_checkpoint=resume_from_checkpoint) |
| File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/trainer.py", line 2325, in train |
| return inner_training_loop( |
| ^^^^^^^^^^^^^^^^^^^^ |
| File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/trainer.py", line 2674, in _inner_training_loop |
| tr_loss_step = self.training_step(model, inputs, num_items_in_batch) |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ |
| File "/workspace/axolotl/src/axolotl/core/trainers/mixins/activation_checkpointing.py", line 46, in training_step |
| return super().training_step(*args, **kwargs) |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ |
| File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/trainer.py", line 4020, in training_step |
| loss = self.compute_loss(model, inputs, num_items_in_batch=num_items_in_batch) |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ |
| File "/workspace/axolotl/src/axolotl/core/trainers/base.py", line 367, in compute_loss |
| return super().compute_loss( |
| ^^^^^^^^^^^^^^^^^^^^^ |
| File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/trainer.py", line 4110, in compute_loss |
| outputs = model(**inputs) |
| ^^^^^^^^^^^^^^^ |
| File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl |
| return self._call_impl(*args, **kwargs) |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ |
| File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1762, in _call_impl |
| return forward_call(*args, **kwargs) |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ |
| File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/accelerate/utils/operations.py", line 818, in forward |
| return model_forward(*args, **kwargs) |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ |
| File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/accelerate/utils/operations.py", line 806, in __call__ |
| return convert_to_fp32(self.model_forward(*args, **kwargs)) |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ |
| File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/torch/amp/autocast_mode.py", line 44, in decorate_autocast |
| return func(*args, **kwargs) |
| ^^^^^^^^^^^^^^^^^^^^^ |
| File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/peft/peft_model.py", line 1850, in forward |
| return self.base_model( |
| ^^^^^^^^^^^^^^^^ |
| File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl |
| return self._call_impl(*args, **kwargs) |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ |
| File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1762, in _call_impl |
| return forward_call(*args, **kwargs) |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ |
| File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/peft/tuners/tuners_utils.py", line 222, in forward |
| return self.model.forward(*args, **kwargs) |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ |
| File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/utils/generic.py", line 918, in wrapper |
| output = func(self, *args, **kwargs) |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^ |
| File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/models/llama/modeling_llama.py", line 459, in forward |
| outputs: BaseModelOutputWithPast = self.model( |
| ^^^^^^^^^^^ |
| File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl |
| return self._call_impl(*args, **kwargs) |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ |
| File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1762, in _call_impl |
| return forward_call(*args, **kwargs) |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ |
| File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/utils/generic.py", line 1064, in wrapper |
| outputs = func(self, *args, **kwargs) |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^ |
| File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/models/llama/modeling_llama.py", line 395, in forward |
| hidden_states = decoder_layer( |
| ^^^^^^^^^^^^^^ |
| File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/modeling_layers.py", line 94, in __call__ |
| return super().__call__(*args, **kwargs) |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ |
| File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl |
| return self._call_impl(*args, **kwargs) |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ |
| File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1762, in _call_impl |
| return forward_call(*args, **kwargs) |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ |
| File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/utils/deprecation.py", line 172, in wrapped_func |
| return func(*args, **kwargs) |
| ^^^^^^^^^^^^^^^^^^^^^ |
| File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/models/llama/modeling_llama.py", line 309, in forward |
| hidden_states = self.mlp(hidden_states) |
| ^^^^^^^^^^^^^^^^^^^^^^^ |
| File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl |
| return self._call_impl(*args, **kwargs) |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ |
| File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1762, in _call_impl |
| return forward_call(*args, **kwargs) |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ |
| File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/models/llama/modeling_llama.py", line 155, in forward |
| down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) |
| ^^^^^^^^^^^^^^^^^ |
| File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl |
| return self._call_impl(*args, **kwargs) |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ |
| File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1762, in _call_impl |
| return forward_call(*args, **kwargs) |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ |
| File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/peft/tuners/lora/layer.py", line 771, in forward |
| result = result + lora_B(lora_A(dropout(x))) * scaling |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^ |
| File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl |
| return self._call_impl(*args, **kwargs) |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ |
| File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1762, in _call_impl |
| return forward_call(*args, **kwargs) |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ |
| File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/torch/nn/modules/linear.py", line 125, in forward |
| return F.linear(input, self.weight, self.bias) |
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ |
| torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 448.00 MiB. GPU 0 has a total capacity of 79.18 GiB of which 298.19 MiB is free. Including non-PyTorch memory, this process has 78.88 GiB memory in use. Of the allocated memory 77.93 GiB is allocated by PyTorch, and 298.73 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https: |
|
0%| | 0/87 [00:02<?, ?it/s] |
|
|
