Upload folder using huggingface_hub

LLaMA-Factory/src/llamafactory/data/converter.py

@@ -50,7 +50,6 @@ class DatasetConverter:
             return None
         else:
             medias = medias[:]
-        logger.warning_rank0_once(f"dataset_attr.load_from: {self.dataset_attr.load_from}")
         if self.dataset_attr.load_from in ["script", "file"]:
             if isinstance(medias[0], str):
                 for i in range(len(medias)):

checkpoints/cold_start/qwen2_5_vl-7b_vilasr_cold_start_8gpu_lora/runs/Oct21_17-55-59_c31/events.out.tfevents.1761084094.c31.1268522.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:885f043ce17c41b93f61f3c56e3b425910f35d8b5f10c50dd790e5ef2591bc16
-size 6796
+oid sha256:fcd413fe9ba7f4f1d735b6fed20a120364ad7686c5096fc45b84e72352acab45
+size 8038

checkpoints/cold_start/qwen2_5_vl-7b_vilasr_cold_start_8gpu_lora/trainer_log.jsonl

@@ -1,2 +1,8 @@
 {"current_steps": 10, "total_steps": 3147, "loss": 0.422, "lr": 2.8571428571428575e-07, "epoch": 0.009529481834425254, "percentage": 0.32, "elapsed_time": "0:23:07", "remaining_time": "5 days, 0:53:15"}
 {"current_steps": 20, "total_steps": 3147, "loss": 0.411, "lr": 6.031746031746032e-07, "epoch": 0.019058963668850508, "percentage": 0.64, "elapsed_time": "0:46:53", "remaining_time": "5 days, 2:11:25"}
+{"current_steps": 30, "total_steps": 3147, "loss": 0.4276, "lr": 9.206349206349208e-07, "epoch": 0.02858844550327576, "percentage": 0.95, "elapsed_time": "1:09:35", "remaining_time": "5 days, 0:31:07"}
+{"current_steps": 40, "total_steps": 3147, "loss": 0.4214, "lr": 1.2380952380952382e-06, "epoch": 0.038117927337701016, "percentage": 1.27, "elapsed_time": "1:34:10", "remaining_time": "5 days, 1:54:57"}
+{"current_steps": 50, "total_steps": 3147, "loss": 0.414, "lr": 1.5555555555555558e-06, "epoch": 0.047647409172126266, "percentage": 1.59, "elapsed_time": "1:57:56", "remaining_time": "5 days, 1:44:55"}
+{"current_steps": 60, "total_steps": 3147, "loss": 0.4039, "lr": 1.8730158730158732e-06, "epoch": 0.05717689100655152, "percentage": 1.91, "elapsed_time": "2:21:36", "remaining_time": "5 days, 1:25:25"}
+{"current_steps": 70, "total_steps": 3147, "loss": 0.418, "lr": 2.1904761904761908e-06, "epoch": 0.06670637284097677, "percentage": 2.22, "elapsed_time": "2:46:27", "remaining_time": "5 days, 1:56:44"}
+{"current_steps": 80, "total_steps": 3147, "loss": 0.4186, "lr": 2.507936507936508e-06, "epoch": 0.07623585467540203, "percentage": 2.54, "elapsed_time": "3:09:39", "remaining_time": "5 days, 1:11:03"}

checkpoints/cold_start/qwen2_5_vl-7b_vilasr_cold_start_8gpu_lora_z2/runs/Oct21_22-00-59_c31/events.out.tfevents.1761098789.c31.1297219.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:faf2f36464713ee200d079fb397bbd02e8345417cca5094ce37d19747d64d499
+size 8256

checkpoints/cold_start/qwen2_5_vl-7b_vilasr_cold_start_8gpu_lora_z2/trainer_log.jsonl

@@ -0,0 +1,9 @@
+{"current_steps": 10, "total_steps": 3147, "loss": 0.4221, "lr": 2.8571428571428575e-07, "epoch": 0.009529481834425254, "percentage": 0.32, "elapsed_time": "0:09:20", "remaining_time": "2 days, 0:51:03"}
+{"current_steps": 20, "total_steps": 3147, "loss": 0.4111, "lr": 6.031746031746032e-07, "epoch": 0.019058963668850508, "percentage": 0.64, "elapsed_time": "0:18:42", "remaining_time": "2 days, 0:44:50"}
+{"current_steps": 30, "total_steps": 3147, "loss": 0.4277, "lr": 9.206349206349208e-07, "epoch": 0.02858844550327576, "percentage": 0.95, "elapsed_time": "0:27:39", "remaining_time": "1 day, 23:54:15"}
+{"current_steps": 40, "total_steps": 3147, "loss": 0.4215, "lr": 1.2380952380952382e-06, "epoch": 0.038117927337701016, "percentage": 1.27, "elapsed_time": "0:37:14", "remaining_time": "2 days, 0:12:06"}
+{"current_steps": 50, "total_steps": 3147, "loss": 0.4139, "lr": 1.5555555555555558e-06, "epoch": 0.047647409172126266, "percentage": 1.59, "elapsed_time": "0:46:37", "remaining_time": "2 days, 0:08:23"}
+{"current_steps": 60, "total_steps": 3147, "loss": 0.404, "lr": 1.8730158730158732e-06, "epoch": 0.05717689100655152, "percentage": 1.91, "elapsed_time": "0:55:56", "remaining_time": "1 day, 23:58:30"}
+{"current_steps": 70, "total_steps": 3147, "loss": 0.4182, "lr": 2.1904761904761908e-06, "epoch": 0.06670637284097677, "percentage": 2.22, "elapsed_time": "1:05:33", "remaining_time": "2 days, 0:01:49"}
+{"current_steps": 80, "total_steps": 3147, "loss": 0.4186, "lr": 2.507936507936508e-06, "epoch": 0.07623585467540203, "percentage": 2.54, "elapsed_time": "1:14:38", "remaining_time": "1 day, 23:41:52"}
+{"current_steps": 90, "total_steps": 3147, "loss": 0.4138, "lr": 2.8253968253968255e-06, "epoch": 0.08576533650982728, "percentage": 2.86, "elapsed_time": "1:23:56", "remaining_time": "1 day, 23:31:18"}

env.yaml

@@ -1,4 +1,3 @@
-name: vilasr
 channels:
   - defaults
   - https://repo.anaconda.com/pkgs/main

requirements.txt

@@ -0,0 +1,235 @@
+absl-py==2.3.1
+accelerate==1.6.0
+aiofiles==24.1.0
+aiohappyeyeballs==2.6.1
+aiohttp==3.13.1
+aiosignal==1.4.0
+airportsdata==20250909
+annotated-types==0.7.0
+antlr4-python3-runtime==4.9.3
+anyio==4.11.0
+astor==0.8.1
+async-timeout==5.0.1
+attrs==25.4.0
+audioread==3.0.1
+av==15.1.0
+blake3==1.0.8
+Brotli==1.1.0
+cachetools==6.2.1
+certifi==2025.8.3
+cffi==2.0.0
+charset-normalizer==3.4.3
+click==8.3.0
+cloudpickle==3.1.1
+codetiming==1.4.0
+compressed-tensors==0.9.1
+contourpy==1.3.2
+cupy-cuda12x==13.6.0
+cycler==0.12.1
+datasets==3.5.0
+decorator==5.2.1
+deepspeed==0.16.9
+depyf==0.18.0
+dill==0.3.8
+diskcache==5.6.3
+distro==1.9.0
+dnspython==2.8.0
+docker-pycreds==0.4.0
+docstring_parser==0.17.0
+einops==0.8.1
+email-validator==2.3.0
+exceptiongroup==1.3.0
+fastapi==0.118.0
+fastapi-cli==0.0.14
+fastapi-cloud-cli==0.3.1
+fastrlock==0.8.3
+ffmpy==0.6.1
+filelock==3.19.1
+fire==0.7.1
+flash_attn==2.5.3
+fonttools==4.60.0
+frozenlist==1.8.0
+fsspec==2024.12.0
+gguf==0.10.0
+gitdb==4.0.12
+GitPython==3.1.45
+google-auth==2.41.1
+google-auth-oauthlib==1.0.0
+gradio==5.45.0
+gradio_client==1.13.0
+groovy==0.1.2
+grpcio==1.75.1
+h11==0.16.0
+hf-xet==1.1.10
+hf_transfer==0.1.9
+hjson==3.1.0
+httpcore==1.0.9
+httptools==0.7.1
+httpx==0.28.1
+huggingface-hub==0.35.3
+idna==3.10
+imageio==2.37.0
+importlib_metadata==8.7.0
+iniconfig==2.3.0
+interegular==0.3.3
+jieba==0.42.1
+Jinja2==3.1.6
+jiter==0.11.0
+joblib==1.5.2
+jsonschema==4.25.1
+jsonschema-specifications==2025.9.1
+kiwisolver==1.4.9
+lark==1.2.2
+lazy_loader==0.4
+librosa==0.11.0
+-e git+https://github.com/hiyouga/LLaMA-Factory.git@2c6aded5d4f4ff23aa1887d16972afb3c2543ac3#egg=llamafactory
+llvmlite==0.43.0
+lm-format-enforcer==0.10.12
+loguru==0.7.3
+Markdown==3.9
+markdown-it-py==4.0.0
+MarkupSafe==3.0.3
+mathruler==0.1.0
+matplotlib==3.10.6
+mdurl==0.1.2
+mistral_common==1.8.5
+modelscope==1.31.0
+mpmath==1.3.0
+msgpack==1.1.2
+msgspec==0.19.0
+multidict==6.7.0
+multiprocess==0.70.16
+nest-asyncio==1.6.0
+networkx==3.4.2
+ninja==1.13.0
+nltk==3.9.1
+numba==0.60.0
+numpy==1.26.4
+nvidia-cublas-cu12==12.6.4.1
+nvidia-cuda-cupti-cu12==12.6.80
+nvidia-cuda-nvrtc-cu12==12.6.77
+nvidia-cuda-runtime-cu12==12.6.77
+nvidia-cudnn-cu12==9.5.1.17
+nvidia-cufft-cu12==11.3.0.4
+nvidia-cufile-cu12==1.13.1.3
+nvidia-curand-cu12==10.3.7.77
+nvidia-cusolver-cu12==11.7.1.2
+nvidia-cusparse-cu12==12.5.4.2
+nvidia-cusparselt-cu12==0.6.3
+nvidia-ml-py==13.580.82
+nvidia-nccl-cu12==2.21.5
+nvidia-nvjitlink-cu12==12.6.85
+nvidia-nvshmem-cu12==3.3.20
+nvidia-nvtx-cu12==12.6.77
+oauthlib==3.3.1
+omegaconf==2.3.0
+openai==1.109.1
+opencv-python-headless==4.11.0.86
+orjson==3.11.3
+outlines==0.1.11
+outlines_core==0.1.26
+packaging==25.0
+pandas==2.3.2
+partial-json-parser==0.2.1.1.post6
+Pebble==5.1.3
+peft==0.17.1
+pillow==11.3.0
+platformdirs==4.5.0
+pluggy==1.6.0
+pooch==1.8.2
+prometheus-fastapi-instrumentator==7.1.0
+prometheus_client==0.23.1
+propcache==0.4.1
+protobuf==3.19.0
+psutil==7.1.0
+py-cpuinfo==9.0.0
+pyarrow==21.0.0
+pyasn1==0.6.1
+pyasn1_modules==0.4.2
+pybind11==3.0.1
+pycountry==24.6.1
+pycparser==2.23
+pydantic==2.10.6
+pydantic-extra-types==2.10.6
+pydantic_core==2.27.2
+pydub==0.25.1
+Pygments==2.19.2
+pylatexenc==2.10
+pyparsing==3.2.5
+pytesseract==0.3.13
+pytest==8.4.2
+python-dateutil==2.9.0.post0
+python-dotenv==1.1.1
+python-multipart==0.0.20
+pytz==2025.2
+PyYAML==6.0.3
+pyzmq==27.1.0
+qwen-vl-utils==0.0.14
+ray==2.47.1
+referencing==0.37.0
+regex==2025.9.18
+requests==2.32.5
+requests-oauthlib==2.0.0
+rich==14.1.0
+rich-toolkit==0.15.1
+rignore==0.7.1
+rouge-chinese==1.0.3
+rpds-py==0.27.1
+rsa==4.9.1
+ruff==0.13.2
+safehttpx==0.1.6
+safetensors==0.5.3
+scikit-image==0.25.2
+scikit-learn==1.7.2
+scipy==1.15.3
+semantic-version==2.10.0
+sentencepiece==0.2.1
+sentry-sdk==2.42.1
+setproctitle==1.3.7
+shellingham==1.5.4
+shtab==1.7.2
+six==1.17.0
+smmap==5.0.2
+sniffio==1.3.1
+soundfile==0.13.1
+soxr==1.0.0
+sse-starlette==3.0.2
+starlette==0.48.0
+sympy==1.13.1
+tabulate==0.9.0
+tensorboard==2.14.0
+tensorboard-data-server==0.7.2
+termcolor==3.1.0
+threadpoolctl==3.6.0
+tifffile==2025.5.10
+tiktoken==0.12.0
+timeout-decorator==0.5.0
+tokenizers==0.21.4
+tomli==2.3.0
+tomlkit==0.13.3
+torch==2.6.0+cu126
+torchaudio==2.6.0+cu126
+torchdata==0.11.0
+torchvision==0.21.0+cu126
+tqdm==4.67.1
+transformers==4.51.1
+triton==3.2.0
+trl==0.9.6
+typer==0.19.2
+typing-inspection==0.4.2
+typing_extensions==4.15.0
+tyro==0.8.14
+tzdata==2025.2
+urllib3==2.5.0
+uvicorn==0.37.0
+uvloop==0.22.1
+vllm==0.7.3
+wandb==0.19.11
+watchfiles==1.1.1
+websockets==15.0.1
+Werkzeug==3.1.3
+xformers==0.0.28.post3
+xgrammar==0.1.11
+xxhash==3.6.0
+yarl==1.22.0
+zipp==3.23.0

train/cold_start/vilasr_full_qwen2.5_vl_7b_8gpu/lora.yaml

@@ -4,6 +4,7 @@ model_name_or_path: ../Qwen2.5-VL-7B-Instruct
 image_max_pixels: 1003520                                             # 1280*28*28
 video_max_pixels: 1003520
 trust_remote_code: true
+flash_attn: fa2
 # infer_backend: vllm
 
 ### method
@@ -13,7 +14,7 @@ finetuning_type: lora
 freeze_vision_tower: true
 freeze_multi_modal_projector: true
 freeze_language_model: false
-deepspeed: train/examples/deepspeed/ds_z3_config.json
+deepspeed: train/examples/deepspeed/ds_z2_config.json
 
 ### dataset
 dataset: vqa_cold_start,maze_cold_start,GPT4Scene_cold_start,SR_91k_cold_start
@@ -27,7 +28,7 @@ dataloader_num_workers: 8 # 4
 # mask_history 是否仅使用当前对话轮次进行训练。default: False
 
 ### output
-output_dir: checkpoints/cold_start/qwen2_5_vl-7b_vilasr_cold_start_8gpu_lora
+output_dir: checkpoints/cold_start/qwen2_5_vl-7b_vilasr_cold_start_8gpu_lora_z2
 logging_steps: 10
 save_steps: 200
 save_total_limit: 1       # new

train/cold_start/vilasr_full_qwen2.5_vl_7b_8gpu/step2_lora.sh.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+#SBATCH --nodes=1
+#SBATCH --ntasks=1
+#SBATCH --wait-all-nodes=1
+#SBATCH --partition=a6000
+#SBATCH --cpus-per-task=64
+#SBATCH -w c31
+export HF_HOME=/mnt/beegfs/dzhu6/.cache
+
+eval "$(/home/dzhu6/miniconda3/bin/conda shell.bash hook)" # init conda
+
+export MASTER_PORT=12763
+
+master_addr=$(scontrol show hostnames "$SLURM_JOB_NODELIST" | head -n 1)
+export MASTER_ADDR=$master_addr
+echo "MASTER_ADDR: $MASTER_ADDR"
+#cpu per task
+echo "SLURM_CPUS_PER_TASK: $SLURM_CPUS_PER_TASK"
+
+source env/bin/activate
+conda env list
+nvidia-smi
+
+export CUDA_VISIBLE_DEVICES=0,1,2,3
+export TOKENIZERS_PARALLELISM=false
+FORCE_TORCHRUN=1 llamafactory-cli train train/cold_start/vilasr_full_qwen2.5_vl_7b_8gpu/lora.yaml