File size: 8,672 Bytes
e24b7c9 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 |
[2025-10-22 10:50:22,584] [DEBUG] [axolotl.utils.config.log_gpu_memory_usage:127] [PID:2355] baseline 0.000GB ()
[2025-10-22 10:50:22,584] [INFO] [axolotl.cli.config.load_cfg:248] [PID:2355] config:
{
"activation_offloading": false,
"adapter": "qlora",
"axolotl_config_path": "config.yaml",
"base_model": "Qwen/Qwen2.5-VL-7B-Instruct",
"base_model_config": "Qwen/Qwen2.5-VL-7B-Instruct",
"batch_size": 8,
"bf16": true,
"capabilities": {
"bf16": true,
"compute_capability": "sm_90",
"fp8": false,
"n_gpu": 1,
"n_node": 1
},
"context_parallel_size": 1,
"dataloader_num_workers": 1,
"dataloader_pin_memory": true,
"dataloader_prefetch_factor": 256,
"dataset_processes": 20,
"datasets": [
{
"message_property_mappings": {
"content": "content",
"role": "role"
},
"path": "/workspace/fine-tuning/data/data.json",
"trust_remote_code": false,
"type": "vl"
}
],
"ddp": false,
"device": "cuda:0",
"dion_rank_fraction": 1.0,
"dion_rank_multiple_of": 1,
"env_capabilities": {
"torch_version": "2.7.1"
},
"eval_batch_size": 1,
"eval_causal_lm_metrics": [
"sacrebleu",
"comet",
"ter",
"chrf"
],
"eval_max_new_tokens": 128,
"eval_table_size": 0,
"experimental_skip_move_to_device": true,
"flash_attention": true,
"fp16": false,
"gradient_accumulation_steps": 8,
"gradient_checkpointing": true,
"gradient_checkpointing_kwargs": {
"use_reentrant": true
},
"include_tkps": true,
"is_multimodal": true,
"learning_rate": 0.0002,
"lisa_layers_attribute": "model.layers",
"load_best_model_at_end": false,
"load_in_4bit": true,
"load_in_8bit": false,
"local_rank": 0,
"lora_alpha": 16,
"lora_dropout": 0.05,
"lora_r": 64,
"lora_target_modules": [
"q_proj",
"v_proj",
"k_proj",
"o_proj",
"gate_proj",
"down_proj",
"up_proj",
"lm_head",
"vision_tower"
],
"loraplus_lr_embedding": 1e-06,
"lr_scheduler": "cosine",
"mean_resizing_embeddings": false,
"micro_batch_size": 1,
"model_config_type": "qwen2_5_vl",
"num_epochs": 3.0,
"optimizer": "paged_adamw_32bit",
"output_dir": "/workspace/fine-tuning/output/mymodel",
"pretrain_multipack_attn": true,
"processor_config": "Qwen/Qwen2.5-VL-7B-Instruct",
"profiler_steps_start": 0,
"qlora_sharded_model_loading": false,
"ray_num_workers": 1,
"resources_per_worker": {
"GPU": 1
},
"sample_packing_bin_size": 200,
"sample_packing_group_size": 100000,
"save_only_model": false,
"save_safetensors": true,
"save_steps": 200,
"save_total_limit": 3,
"sequence_len": 2048,
"shuffle_before_merging_datasets": false,
"shuffle_merged_datasets": true,
"skip_prepare_dataset": false,
"streaming_multipack_buffer_size": 10000,
"strict": false,
"tensor_parallel_size": 1,
"tiled_mlp_use_original_mlp": true,
"tokenizer_config": "Qwen/Qwen2.5-VL-7B-Instruct",
"tokenizer_save_jinja_files": true,
"torch_compile": false,
"torch_dtype": "torch.bfloat16",
"train_on_inputs": false,
"trl": {
"log_completions": false,
"mask_truncated_completions": false,
"ref_model_mixup_alpha": 0.9,
"ref_model_sync_steps": 64,
"scale_rewards": true,
"sync_ref_model": false,
"use_vllm": false,
"vllm_server_host": "0.0.0.0",
"vllm_server_port": 8000
},
"trust_remote_code": true,
"use_ray": false,
"val_set_size": 0.0,
"vllm": {
"device": "auto",
"dtype": "auto",
"gpu_memory_utilization": 0.9,
"host": "0.0.0.0",
"port": 8000
},
"weight_decay": 0.0,
"world_size": 1
}
[2025-10-22 10:50:23,548] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:278] [PID:2355] EOS: 151645 / <|im_end|>
[2025-10-22 10:50:23,548] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:279] [PID:2355] BOS: None / None
[2025-10-22 10:50:23,548] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:280] [PID:2355] PAD: 151643 / <|endoftext|>
[2025-10-22 10:50:23,548] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:281] [PID:2355] UNK: None / None
[2025-10-22 10:50:23,549] [INFO] [axolotl.utils.data.shared.load_preprocessed_dataset:476] [PID:2355] Unable to find prepared dataset in last_run_prepared/b27c6ba83f346393518eaef14d5ae408
[2025-10-22 10:50:23,549] [INFO] [axolotl.utils.data.sft._load_raw_datasets:320] [PID:2355] Loading raw datasets...
[2025-10-22 10:50:23,549] [WARNING] [axolotl.utils.data.sft._load_raw_datasets:322] [PID:2355] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`.
[2025-10-22 10:50:23,988] [INFO] [axolotl.utils.data.wrappers.get_dataset_wrapper:87] [PID:2355] Loading dataset: /workspace/fine-tuning/data/data.json with base_type: vl and prompt_style: None
[2025-10-22 10:50:23,989] [ERROR] [axolotl.utils.data.wrappers.handle_unknown_dataset_strategy:53] [PID:2355] unhandled prompt tokenization strategy: vl.
Traceback (most recent call last):
File "<frozen runpy>", line 198, in _run_module_as_main
File "<frozen runpy>", line 88, in _run_code
File "/workspace/axolotl/src/axolotl/cli/train.py", line 121, in <module>
fire.Fire(do_cli)
File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/fire/core.py", line 135, in Fire
component_trace = _Fire(component, args, parsed_flag_args, context, name)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/fire/core.py", line 468, in _Fire
component, remaining_args = _CallAndUpdateTrace(
^^^^^^^^^^^^^^^^^^^^
File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/fire/core.py", line 684, in _CallAndUpdateTrace
component = fn(*varargs, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/axolotl/src/axolotl/cli/train.py", line 88, in do_cli
return do_train(parsed_cfg, parsed_cli_args)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/axolotl/src/axolotl/cli/train.py", line 43, in do_train
dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/axolotl/src/axolotl/common/datasets.py", line 59, in load_datasets
train_dataset, eval_dataset, total_num_steps, prompters = prepare_datasets(
^^^^^^^^^^^^^^^^^
File "/workspace/axolotl/src/axolotl/utils/data/utils.py", line 50, in wrapper
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/workspace/axolotl/src/axolotl/utils/data/sft.py", line 65, in prepare_datasets
return _prepare_standard_dataset(cfg, tokenizer, processor)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/axolotl/src/axolotl/utils/data/sft.py", line 98, in _prepare_standard_dataset
train_dataset, eval_dataset, prompters = loader.load(_load_datasets)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/axolotl/src/axolotl/utils/data/lock.py", line 38, in load
result = load_fn()
^^^^^^^^^
File "/workspace/axolotl/src/axolotl/utils/data/sft.py", line 77, in _load_datasets
train_dataset, eval_dataset, prompters = _load_and_prepare_datasets(
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/axolotl/src/axolotl/utils/data/sft.py", line 503, in _load_and_prepare_datasets
dataset, prompters = _load_tokenized_prepared_datasets(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/axolotl/src/axolotl/utils/data/sft.py", line 299, in _load_tokenized_prepared_datasets
dataset, prompters = _load_raw_datasets(
^^^^^^^^^^^^^^^^^^^
File "/workspace/axolotl/src/axolotl/utils/data/sft.py", line 331, in _load_raw_datasets
dataset_wrapper, dataset_prompter = _load_and_process_single_dataset(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/axolotl/src/axolotl/utils/data/sft.py", line 401, in _load_and_process_single_dataset
dataset_wrapper, dataset_prompter = get_dataset_wrapper(
^^^^^^^^^^^^^^^^^^^^
File "/workspace/axolotl/src/axolotl/utils/data/wrappers.py", line 131, in get_dataset_wrapper
handle_unknown_dataset_strategy(dataset_config)
File "/workspace/axolotl/src/axolotl/utils/data/wrappers.py", line 54, in handle_unknown_dataset_strategy
raise ValueError(error_message)
ValueError: unhandled prompt tokenization strategy: vl.
|