Upload folder using huggingface_hub

.gitattributes

@@ -35,3 +35,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 run-20250329_010934-3x35hjks/run-3x35hjks.wandb filter=lfs diff=lfs merge=lfs -text
 run-20250329_012205-co1ecmky/run-co1ecmky.wandb filter=lfs diff=lfs merge=lfs -text
+offline-run-20250328_203038-ni3gydc1/run-ni3gydc1.wandb filter=lfs diff=lfs merge=lfs -text
+offline-run-20250328_203844-gc2qytpj/run-gc2qytpj.wandb filter=lfs diff=lfs merge=lfs -text

debug-internal.log

@@ -1,7 +1,7 @@
-{"time":"2025-03-29T01:22:05.252520599+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"../outputs/test_7B/wandb/run-20250329_012205-co1ecmky/logs/debug-core.log"}
-{"time":"2025-03-29T01:22:05.467254306+08:00","level":"INFO","msg":"created new stream","id":"co1ecmky"}
-{"time":"2025-03-29T01:22:05.467309548+08:00","level":"INFO","msg":"stream: started","id":"co1ecmky"}
-{"time":"2025-03-29T01:22:05.467331707+08:00","level":"INFO","msg":"handler: started","stream_id":"co1ecmky"}
-{"time":"2025-03-29T01:22:05.467333162+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"co1ecmky"}
-{"time":"2025-03-29T01:22:05.467336174+08:00","level":"INFO","msg":"sender: started","stream_id":"co1ecmky"}
-{"time":"2025-03-29T01:22:05.772490021+08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2025-03-28T20:38:44.692258573+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"../outputs/test_13B/wandb/offline-run-20250328_203844-gc2qytpj/logs/debug-core.log"}
+{"time":"2025-03-28T20:38:44.90653769+08:00","level":"INFO","msg":"created new stream","id":"gc2qytpj"}
+{"time":"2025-03-28T20:38:44.90660149+08:00","level":"INFO","msg":"stream: started","id":"gc2qytpj"}
+{"time":"2025-03-28T20:38:44.906615038+08:00","level":"INFO","msg":"handler: started","stream_id":"gc2qytpj"}
+{"time":"2025-03-28T20:38:44.906633311+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"gc2qytpj"}
+{"time":"2025-03-28T20:38:44.90663431+08:00","level":"INFO","msg":"sender: started","stream_id":"gc2qytpj"}
+{"time":"2025-03-28T20:38:44.910439284+08:00","level":"INFO","msg":"Starting system monitor"}

debug.log

@@ -1,24 +1,26 @@
-2025-03-29 01:22:05,246 INFO    MainThread:104999 [wandb_setup.py:_flush():67] Current SDK version is 0.19.8
-2025-03-29 01:22:05,246 INFO    MainThread:104999 [wandb_setup.py:_flush():67] Configure stats pid to 104999
-2025-03-29 01:22:05,246 INFO    MainThread:104999 [wandb_setup.py:_flush():67] Loading settings from /home/yangyaodong/.config/wandb/settings
-2025-03-29 01:22:05,246 INFO    MainThread:104999 [wandb_setup.py:_flush():67] Loading settings from /aifs4su/yaodong/hantao/align-anything/scripts/wandb/settings
-2025-03-29 01:22:05,246 INFO    MainThread:104999 [wandb_setup.py:_flush():67] Loading settings from environment variables
-2025-03-29 01:22:05,246 INFO    MainThread:104999 [wandb_init.py:setup_run_log_directory():647] Logging user logs to ../outputs/test_7B/wandb/run-20250329_012205-co1ecmky/logs/debug.log
-2025-03-29 01:22:05,246 INFO    MainThread:104999 [wandb_init.py:setup_run_log_directory():648] Logging internal logs to ../outputs/test_7B/wandb/run-20250329_012205-co1ecmky/logs/debug-internal.log
-2025-03-29 01:22:05,246 INFO    MainThread:104999 [wandb_init.py:init():761] calling init triggers
-2025-03-29 01:22:05,246 INFO    MainThread:104999 [wandb_init.py:init():766] wandb.init called with sweep_config: {}
-config: {'train_cfgs': {'save_checkpoint': False, 'load_checkpoint': False, 'ds_cfgs': 'ds_z3_config.json', 'epochs': 3, 'seed': 42, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 1, 'gradient_accumulation_steps': 16, 'gradient_checkpointing': True, 'learning_rate': 2e-05, 'lr_scheduler_type': 'cosine', 'lr_warmup_ratio': 0.03, 'weight_decay': 0.0, 'adam_betas': [0.9, 0.95], 'adam_epsilon': 1e-08, 'bf16': True, 'fp16': False, 'eval_strategy': 'epoch', 'eval_interval': 10, 'freeze_mm_proj': False, 'freeze_vision_tower': True, 'freeze_language_model': False, 'max_grad_norm': 1.0}, 'data_cfgs': {'load_multi_datasets': False, 'train_datasets': '/aifs4su/yaodong/hantao/datasets/MMInstruct-GPT4V_mistral-7b_cosi_cut/merged/top1-10', 'train_template': 'MM_TI2T_LLAVA', 'train_size': {}, 'train_split': 'train', 'train_name': 'text-image-to-text', 'train_data_files': {}, 'train_optional_args': [], 'eval_datasets': {}, 'eval_template': {}, 'eval_name': {}, 'eval_size': {}, 'eval_split': {}, 'eval_subset': {}, 'eval_data_files': {}, 'eval_optional_args': []}, 'logger_cfgs': {'log_type': 'wandb', 'log_project': 'align-anything', 'log_run_name': 'sft', 'output_dir': '../outputs/test_7B', 'cache_dir': {}, 'save_total_limit': 6}, 'model_cfgs': {'model_name_or_path': '/aifs4su/yaodong/hantao/models/llava-v1.6-mistral-7b-hf', 'trust_remote_code': True, 'model_max_length': 2048}, 'special_tokens': {}, '_wandb': {}}
-2025-03-29 01:22:05,246 INFO    MainThread:104999 [wandb_init.py:init():784] starting backend
-2025-03-29 01:22:05,246 INFO    MainThread:104999 [wandb_init.py:init():788] sending inform_init request
-2025-03-29 01:22:05,249 INFO    MainThread:104999 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
-2025-03-29 01:22:05,249 INFO    MainThread:104999 [wandb_init.py:init():798] backend started and connected
-2025-03-29 01:22:05,251 INFO    MainThread:104999 [wandb_init.py:init():891] updated telemetry
-2025-03-29 01:22:05,262 INFO    MainThread:104999 [wandb_init.py:init():915] communicating run to backend with 90.0 second timeout
-2025-03-29 01:22:05,770 INFO    MainThread:104999 [wandb_init.py:init():990] starting run threads in backend
-2025-03-29 01:22:05,989 INFO    MainThread:104999 [wandb_run.py:_console_start():2375] atexit reg
-2025-03-29 01:22:05,990 INFO    MainThread:104999 [wandb_run.py:_redirect():2227] redirect: wrap_raw
-2025-03-29 01:22:05,990 INFO    MainThread:104999 [wandb_run.py:_redirect():2292] Wrapping output streams.
-2025-03-29 01:22:05,990 INFO    MainThread:104999 [wandb_run.py:_redirect():2315] Redirects installed.
-2025-03-29 01:22:05,992 INFO    MainThread:104999 [wandb_init.py:init():1032] run started, returning control to user process
-2025-03-29 01:26:51,278 INFO    MainThread:104999 [wandb_run.py:_finish():2112] finishing run htlou/align-anything/co1ecmky
-2025-03-29 01:26:51,278 INFO    MainThread:104999 [wandb_run.py:_atexit_cleanup():2340] got exitcode: 0
+2025-03-28 20:38:44,447 INFO    MainThread:3871311 [wandb_setup.py:_flush():67] Current SDK version is 0.19.8
+2025-03-28 20:38:44,447 INFO    MainThread:3871311 [wandb_setup.py:_flush():67] Configure stats pid to 3871311
+2025-03-28 20:38:44,447 INFO    MainThread:3871311 [wandb_setup.py:_flush():67] Loading settings from /home/yangyaodong/.config/wandb/settings
+2025-03-28 20:38:44,447 INFO    MainThread:3871311 [wandb_setup.py:_flush():67] Loading settings from /aifs4su/yaodong/hantao/align-anything/scripts/wandb/settings
+2025-03-28 20:38:44,447 INFO    MainThread:3871311 [wandb_setup.py:_flush():67] Loading settings from environment variables
+2025-03-28 20:38:44,447 INFO    MainThread:3871311 [wandb_init.py:setup_run_log_directory():647] Logging user logs to ../outputs/test_13B/wandb/offline-run-20250328_203844-gc2qytpj/logs/debug.log
+2025-03-28 20:38:44,447 INFO    MainThread:3871311 [wandb_init.py:setup_run_log_directory():648] Logging internal logs to ../outputs/test_13B/wandb/offline-run-20250328_203844-gc2qytpj/logs/debug-internal.log
+2025-03-28 20:38:44,447 INFO    MainThread:3871311 [wandb_init.py:init():761] calling init triggers
+2025-03-28 20:38:44,447 INFO    MainThread:3871311 [wandb_init.py:init():766] wandb.init called with sweep_config: {}
+config: {'train_cfgs': {'save_checkpoint': True, 'load_checkpoint': False, 'ds_cfgs': 'ds_z3_config.json', 'epochs': 3, 'seed': 42, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 1, 'gradient_accumulation_steps': 1, 'gradient_checkpointing': True, 'learning_rate': 1e-06, 'lr_scheduler_type': 'cosine', 'lr_warmup_ratio': 0.03, 'weight_decay': 0.0, 'adam_betas': [0.9, 0.95], 'bf16': True, 'fp16': False, 'eval_strategy': 'epoch', 'eval_interval': 10, 'regularization': 0.001, 'scale_coeff': 0.1, 'freeze_mm_proj': False, 'freeze_vision_tower': True, 'freeze_language_model': False}, 'data_cfgs': {'train_datasets': '/aifs4su/yaodong/hantao/datasets/AA_preference_vicuna-13b_cosi_cut/merged/top1-10', 'train_template': 'AA_TI2T_LLAVA', 'train_size': {}, 'train_split': 'train', 'train_name': 'text-image-to-text', 'train_data_files': {}, 'train_optional_args': [], 'eval_datasets': {}, 'eval_template': {}, 'eval_size': {}, 'eval_split': {}, 'eval_subset': {}, 'eval_data_files': {}, 'eval_optional_args': []}, 'logger_cfgs': {'log_type': 'wandb', 'log_project': 'align-anything', 'log_run_name': 'dpo', 'output_dir': '../outputs/test_13B', 'cache_dir': {}, 'save_total_limit': 6}, 'model_cfgs': {'model_name_or_path': '/aifs4su/yaodong/hantao/models/llava-v1.6-vicuna-13b-hf', 'trust_remote_code': True, 'model_max_length': 2048}, 'special_tokens': {}, '_wandb': {}}
+2025-03-28 20:38:44,447 INFO    MainThread:3871311 [wandb_init.py:init():784] starting backend
+2025-03-28 20:38:44,657 INFO    MainThread:3871311 [wandb_init.py:init():788] sending inform_init request
+2025-03-28 20:38:44,660 INFO    MainThread:3871311 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-03-28 20:38:44,660 INFO    MainThread:3871311 [wandb_init.py:init():798] backend started and connected
+2025-03-28 20:38:44,662 INFO    MainThread:3871311 [wandb_init.py:init():891] updated telemetry
+2025-03-28 20:38:44,672 INFO    MainThread:3871311 [wandb_init.py:init():915] communicating run to backend with 90.0 second timeout
+2025-03-28 20:38:44,908 INFO    MainThread:3871311 [wandb_init.py:init():990] starting run threads in backend
+2025-03-28 20:38:45,071 INFO    MainThread:3871311 [wandb_run.py:_console_start():2375] atexit reg
+2025-03-28 20:38:45,071 INFO    MainThread:3871311 [wandb_run.py:_redirect():2227] redirect: wrap_raw
+2025-03-28 20:38:45,071 INFO    MainThread:3871311 [wandb_run.py:_redirect():2292] Wrapping output streams.
+2025-03-28 20:38:45,071 INFO    MainThread:3871311 [wandb_run.py:_redirect():2315] Redirects installed.
+2025-03-28 20:38:45,074 INFO    MainThread:3871311 [wandb_init.py:init():1032] run started, returning control to user process
+2025-03-28 20:47:40,230 INFO    MainThread:3871311 [wandb_run.py:_finish():2112] finishing run align-anything/gc2qytpj
+2025-03-28 20:47:40,230 INFO    MainThread:3871311 [wandb_run.py:_atexit_cleanup():2340] got exitcode: 0
+2025-03-28 20:47:40,230 INFO    MainThread:3871311 [wandb_run.py:_restore():2322] restore
+2025-03-28 20:47:40,230 INFO    MainThread:3871311 [wandb_run.py:_restore():2328] restore done

offline-run-20250328_203038-ni3gydc1/files/requirements.txt

@@ -0,0 +1,167 @@
+maskrcnn_benchmark==0.0.0
+webdataset==0.2.111
+websockets==15.0.1
+typer==0.15.2
+blobfile==3.0.0
+pooch==1.8.2
+python-dateutil==2.9.0.post0
+gmpy2==2.2.1
+httpcore==1.0.7
+charset-normalizer==3.3.2
+torchlibrosa==0.1.0
+multiprocess==0.70.16
+Werkzeug==3.1.3
+aiofiles==23.2.1
+six==1.17.0
+typing_extensions==4.12.2
+psutil==7.0.0
+frozenlist==1.5.0
+einops==0.8.1
+flash_attn==2.7.4.post1
+PySocks==1.7.1
+regex==2024.11.6
+markdown-it-py==3.0.0
+ruff==0.11.2
+docker-pycreds==0.4.0
+protobuf==5.29.4
+resampy==0.4.3
+aiohappyeyeballs==2.6.1
+httpx==0.28.1
+encodec==0.1.1
+ffmpy==0.5.0
+mkl_random==1.2.8
+soxr==0.5.0.post1
+absl-py==2.2.1
+networkx==3.4.2
+h5py==3.13.0
+hjson==3.1.0
+tensorboard==2.19.0
+aiosignal==1.3.2
+pip==25.0
+triton==3.1.0
+zipp==3.21.0
+ftfy==6.3.1
+attrs==25.3.0
+requests==2.32.3
+progressbar==2.5
+sniffio==1.3.1
+lxml==5.3.1
+starlette==0.46.1
+Markdown==3.7
+mdurl==0.1.2
+torchaudio==2.5.1
+safetensors==0.5.3
+opencv-python==4.6.0.66
+torchvision==0.20.1
+shellingham==1.5.4
+gradio==5.23.1
+timm==1.0.15
+multidict==6.2.0
+semantic-version==2.10.0
+numba==0.60.0
+gradio_client==1.8.0
+pydantic_core==2.33.0
+dill==0.3.8
+msgpack==1.1.0
+sentry-sdk==2.24.1
+grpcio==1.71.0
+cffi==1.17.1
+PyYAML==6.0.2
+tensorboard-data-server==0.7.2
+fastapi==0.115.12
+lazy_loader==0.4
+mkl_fft==1.3.11
+annotated-types==0.7.0
+scikit-learn==1.6.1
+wget==3.2
+setuptools==75.8.0
+certifi==2025.1.31
+click==8.1.8
+laion_clap==1.1.5
+Pygments==2.19.1
+tomlkit==0.13.2
+idna==3.7
+propcache==0.3.1
+platformdirs==4.3.7
+align-anything==0.0.1.dev0
+deepspeed==0.16.5
+smmap==5.0.2
+pillow==11.1.0
+typing-inspection==0.4.0
+braceexpand==0.1.7
+decorator==5.2.1
+pandas==2.2.3
+huggingface-hub==0.29.3
+pyarrow==19.0.1
+tokenizers==0.21.1
+GitPython==3.1.44
+xxhash==3.5.0
+packaging==24.2
+numpy==1.23.4
+setproctitle==1.3.5
+llvmlite==0.43.0
+tiktoken==0.9.0
+mpmath==1.3.0
+nvidia-ml-py==12.570.86
+pydantic==2.11.0
+datasets==3.5.0
+librosa==0.11.0
+frechet_audio_distance==0.3.1
+sympy==1.13.1
+safehttpx==0.1.6
+Jinja2==3.1.6
+h11==0.14.0
+aiohttp==3.11.14
+diffusers==0.32.2
+tqdm==4.67.1
+filelock==3.13.1
+transformers==4.51.0.dev0
+scipy==1.10.1
+audioread==3.0.1
+sentencepiece==0.2.0
+pytz==2025.2
+tzdata==2025.2
+python-multipart==0.0.20
+urllib3==2.3.0
+pycryptodomex==3.22.0
+yarl==1.18.3
+pydub==0.25.1
+pycparser==2.22
+soundfile==0.13.1
+wcwidth==0.2.13
+groovy==0.1.2
+torch==2.5.1
+anyio==4.9.0
+wandb==0.19.8
+joblib==1.4.2
+fsspec==2024.12.0
+peft==0.15.1
+accelerate==1.5.2
+py-cpuinfo==9.0.0
+uvicorn==0.34.0
+orjson==3.10.16
+Brotli==1.0.9
+rich==13.9.4
+importlib_metadata==8.6.1
+ninja==1.11.1.4
+wheel==0.45.1
+MarkupSafe==3.0.2
+threadpoolctl==3.6.0
+gitdb==4.0.12
+mkl-service==2.4.0
+typing_extensions==4.12.2
+tomli==2.0.1
+zipp==3.19.2
+wheel==0.43.0
+jaraco.text==3.12.1
+packaging==24.2
+autocommand==2.2.2
+jaraco.functools==4.0.1
+jaraco.collections==5.1.0
+platformdirs==4.2.2
+more-itertools==10.3.0
+inflect==7.3.1
+jaraco.context==5.3.0
+typeguard==4.3.0
+backports.tarfile==1.2.0
+importlib_metadata==8.0.0

offline-run-20250328_203038-ni3gydc1/files/wandb-metadata.json

@@ -0,0 +1,105 @@
+{
+  "os": "Linux-5.15.0-1040-nvidia-x86_64-with-glibc2.35",
+  "python": "CPython 3.11.11",
+  "startedAt": "2025-03-28T12:30:39.058990Z",
+  "args": [
+    "--local_rank=0",
+    "--model_name_or_path",
+    "/aifs4su/yaodong/hantao/models/llava-v1.6-vicuna-13b-hf",
+    "--train_datasets",
+    "/aifs4su/yaodong/hantao/datasets/AA_preference_vicuna-13b_cosi_cut/merged/top1-10",
+    "--train_template",
+    "AA_TI2T_LLAVA",
+    "--train_split",
+    "train",
+    "--train_name",
+    "text-image-to-text",
+    "--output_dir",
+    "../outputs/test_13B",
+    "--save_total_limit",
+    "3",
+    "--train_batch_size",
+    "1",
+    "--epochs",
+    "2"
+  ],
+  "program": "-m align_anything.trainers.text_image_to_text.dpo",
+  "git": {
+    "remote": "git@github.com-hantao:PKU-Alignment/align-anything.git",
+    "commit": "106588f9802757a3283c1aff1f33ea9afd737f31"
+  },
+  "root": "../outputs/test_13B",
+  "host": "dgx-092",
+  "executable": "/aifs4su/yaodong/miniconda3/envs/hantao_llama/bin/python",
+  "cpu_count": 112,
+  "cpu_count_logical": 224,
+  "gpu": "NVIDIA H800",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1888556142592",
+      "used": "148592041984"
+    }
+  },
+  "memory": {
+    "total": "2164195454976"
+  },
+  "cpu": {
+    "count": 112,
+    "countLogical": 224
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA H800",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H800",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H800",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H800",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H800",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H800",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H800",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H800",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    }
+  ],
+  "slurm": {
+    "conf": "/cm/shared/apps/slurm/var/etc/slurm/slurm.conf"
+  },
+  "cudaVersion": "12.2"
+}

offline-run-20250328_203038-ni3gydc1/logs/debug-core.log

@@ -0,0 +1,7 @@
+{"time":"2025-03-28T20:30:38.872129247+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpmsjq2qrj/port-3856880.txt","pid":3856880,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false}
+{"time":"2025-03-28T20:30:38.873015217+08:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":3856880}
+{"time":"2025-03-28T20:30:38.873005488+08:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":44091,"Zone":""}}
+{"time":"2025-03-28T20:30:39.052962944+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:34304"}
+{"time":"2025-03-28T20:30:39.059989776+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"ni3gydc1","id":"127.0.0.1:34304"}
+{"time":"2025-03-28T20:30:39.456558723+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"ni3gydc1","id":"127.0.0.1:34304"}
+{"time":"2025-03-28T20:36:33.911468496+08:00","level":"INFO","msg":"received shutdown signal","signal":15}

offline-run-20250328_203038-ni3gydc1/logs/debug-internal.log

@@ -0,0 +1,7 @@
+{"time":"2025-03-28T20:30:39.242295984+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"../outputs/test_13B/wandb/offline-run-20250328_203038-ni3gydc1/logs/debug-core.log"}
+{"time":"2025-03-28T20:30:39.456487825+08:00","level":"INFO","msg":"created new stream","id":"ni3gydc1"}
+{"time":"2025-03-28T20:30:39.456549716+08:00","level":"INFO","msg":"stream: started","id":"ni3gydc1"}
+{"time":"2025-03-28T20:30:39.456572806+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"ni3gydc1"}
+{"time":"2025-03-28T20:30:39.456577964+08:00","level":"INFO","msg":"handler: started","stream_id":"ni3gydc1"}
+{"time":"2025-03-28T20:30:39.456615028+08:00","level":"INFO","msg":"sender: started","stream_id":"ni3gydc1"}
+{"time":"2025-03-28T20:30:39.462148507+08:00","level":"INFO","msg":"Starting system monitor"}

offline-run-20250328_203038-ni3gydc1/logs/debug.log

@@ -0,0 +1,26 @@
+2025-03-28 20:30:38,838 INFO    MainThread:3856880 [wandb_setup.py:_flush():67] Current SDK version is 0.19.8
+2025-03-28 20:30:38,839 INFO    MainThread:3856880 [wandb_setup.py:_flush():67] Configure stats pid to 3856880
+2025-03-28 20:30:38,839 INFO    MainThread:3856880 [wandb_setup.py:_flush():67] Loading settings from /home/yangyaodong/.config/wandb/settings
+2025-03-28 20:30:38,839 INFO    MainThread:3856880 [wandb_setup.py:_flush():67] Loading settings from /aifs4su/yaodong/hantao/align-anything/scripts/wandb/settings
+2025-03-28 20:30:38,839 INFO    MainThread:3856880 [wandb_setup.py:_flush():67] Loading settings from environment variables
+2025-03-28 20:30:38,839 INFO    MainThread:3856880 [wandb_init.py:setup_run_log_directory():647] Logging user logs to ../outputs/test_13B/wandb/offline-run-20250328_203038-ni3gydc1/logs/debug.log
+2025-03-28 20:30:38,839 INFO    MainThread:3856880 [wandb_init.py:setup_run_log_directory():648] Logging internal logs to ../outputs/test_13B/wandb/offline-run-20250328_203038-ni3gydc1/logs/debug-internal.log
+2025-03-28 20:30:38,839 INFO    MainThread:3856880 [wandb_init.py:init():761] calling init triggers
+2025-03-28 20:30:38,839 INFO    MainThread:3856880 [wandb_init.py:init():766] wandb.init called with sweep_config: {}
+config: {'train_cfgs': {'save_checkpoint': True, 'load_checkpoint': False, 'ds_cfgs': 'ds_z3_config.json', 'epochs': 2, 'seed': 42, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 1, 'gradient_accumulation_steps': 1, 'gradient_checkpointing': True, 'learning_rate': 1e-06, 'lr_scheduler_type': 'cosine', 'lr_warmup_ratio': 0.03, 'weight_decay': 0.0, 'adam_betas': [0.9, 0.95], 'bf16': True, 'fp16': False, 'eval_strategy': 'epoch', 'eval_interval': 10, 'regularization': 0.001, 'scale_coeff': 0.1, 'freeze_mm_proj': False, 'freeze_vision_tower': True, 'freeze_language_model': False}, 'data_cfgs': {'train_datasets': '/aifs4su/yaodong/hantao/datasets/AA_preference_vicuna-13b_cosi_cut/merged/top1-10', 'train_template': 'AA_TI2T_LLAVA', 'train_size': {}, 'train_split': 'train', 'train_name': 'text-image-to-text', 'train_data_files': {}, 'train_optional_args': [], 'eval_datasets': {}, 'eval_template': {}, 'eval_size': {}, 'eval_split': {}, 'eval_subset': {}, 'eval_data_files': {}, 'eval_optional_args': []}, 'logger_cfgs': {'log_type': 'wandb', 'log_project': 'align-anything', 'log_run_name': 'dpo', 'output_dir': '../outputs/test_13B', 'cache_dir': {}, 'save_total_limit': 3}, 'model_cfgs': {'model_name_or_path': '/aifs4su/yaodong/hantao/models/llava-v1.6-vicuna-13b-hf', 'trust_remote_code': True, 'model_max_length': 2048}, 'special_tokens': {}, '_wandb': {}}
+2025-03-28 20:30:38,839 INFO    MainThread:3856880 [wandb_init.py:init():784] starting backend
+2025-03-28 20:30:39,053 INFO    MainThread:3856880 [wandb_init.py:init():788] sending inform_init request
+2025-03-28 20:30:39,058 INFO    MainThread:3856880 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-03-28 20:30:39,058 INFO    MainThread:3856880 [wandb_init.py:init():798] backend started and connected
+2025-03-28 20:30:39,060 INFO    MainThread:3856880 [wandb_init.py:init():891] updated telemetry
+2025-03-28 20:30:39,069 INFO    MainThread:3856880 [wandb_init.py:init():915] communicating run to backend with 90.0 second timeout
+2025-03-28 20:30:39,458 INFO    MainThread:3856880 [wandb_init.py:init():990] starting run threads in backend
+2025-03-28 20:30:39,624 INFO    MainThread:3856880 [wandb_run.py:_console_start():2375] atexit reg
+2025-03-28 20:30:39,624 INFO    MainThread:3856880 [wandb_run.py:_redirect():2227] redirect: wrap_raw
+2025-03-28 20:30:39,624 INFO    MainThread:3856880 [wandb_run.py:_redirect():2292] Wrapping output streams.
+2025-03-28 20:30:39,624 INFO    MainThread:3856880 [wandb_run.py:_redirect():2315] Redirects installed.
+2025-03-28 20:30:39,627 INFO    MainThread:3856880 [wandb_init.py:init():1032] run started, returning control to user process
+2025-03-28 20:36:33,901 INFO    MainThread:3856880 [wandb_run.py:_finish():2112] finishing run align-anything/ni3gydc1
+2025-03-28 20:36:33,902 INFO    MainThread:3856880 [wandb_run.py:_atexit_cleanup():2340] got exitcode: 0
+2025-03-28 20:36:33,902 INFO    MainThread:3856880 [wandb_run.py:_restore():2322] restore
+2025-03-28 20:36:33,902 INFO    MainThread:3856880 [wandb_run.py:_restore():2328] restore done

offline-run-20250328_203038-ni3gydc1/run-ni3gydc1.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7551a30274cb4766f54360f45c34e659bd36728d49e2f3ed7d96d44362f0c1ca
+size 229376

offline-run-20250328_203844-gc2qytpj/files/requirements.txt

@@ -0,0 +1,167 @@
+maskrcnn_benchmark==0.0.0
+webdataset==0.2.111
+websockets==15.0.1
+typer==0.15.2
+blobfile==3.0.0
+pooch==1.8.2
+python-dateutil==2.9.0.post0
+gmpy2==2.2.1
+httpcore==1.0.7
+charset-normalizer==3.3.2
+torchlibrosa==0.1.0
+multiprocess==0.70.16
+Werkzeug==3.1.3
+aiofiles==23.2.1
+six==1.17.0
+typing_extensions==4.12.2
+psutil==7.0.0
+frozenlist==1.5.0
+einops==0.8.1
+flash_attn==2.7.4.post1
+PySocks==1.7.1
+regex==2024.11.6
+markdown-it-py==3.0.0
+ruff==0.11.2
+docker-pycreds==0.4.0
+protobuf==5.29.4
+resampy==0.4.3
+aiohappyeyeballs==2.6.1
+httpx==0.28.1
+encodec==0.1.1
+ffmpy==0.5.0
+mkl_random==1.2.8
+soxr==0.5.0.post1
+absl-py==2.2.1
+networkx==3.4.2
+h5py==3.13.0
+hjson==3.1.0
+tensorboard==2.19.0
+aiosignal==1.3.2
+pip==25.0
+triton==3.1.0
+zipp==3.21.0
+ftfy==6.3.1
+attrs==25.3.0
+requests==2.32.3
+progressbar==2.5
+sniffio==1.3.1
+lxml==5.3.1
+starlette==0.46.1
+Markdown==3.7
+mdurl==0.1.2
+torchaudio==2.5.1
+safetensors==0.5.3
+opencv-python==4.6.0.66
+torchvision==0.20.1
+shellingham==1.5.4
+gradio==5.23.1
+timm==1.0.15
+multidict==6.2.0
+semantic-version==2.10.0
+numba==0.60.0
+gradio_client==1.8.0
+pydantic_core==2.33.0
+dill==0.3.8
+msgpack==1.1.0
+sentry-sdk==2.24.1
+grpcio==1.71.0
+cffi==1.17.1
+PyYAML==6.0.2
+tensorboard-data-server==0.7.2
+fastapi==0.115.12
+lazy_loader==0.4
+mkl_fft==1.3.11
+annotated-types==0.7.0
+scikit-learn==1.6.1
+wget==3.2
+setuptools==75.8.0
+certifi==2025.1.31
+click==8.1.8
+laion_clap==1.1.5
+Pygments==2.19.1
+tomlkit==0.13.2
+idna==3.7
+propcache==0.3.1
+platformdirs==4.3.7
+align-anything==0.0.1.dev0
+deepspeed==0.16.5
+smmap==5.0.2
+pillow==11.1.0
+typing-inspection==0.4.0
+braceexpand==0.1.7
+decorator==5.2.1
+pandas==2.2.3
+huggingface-hub==0.29.3
+pyarrow==19.0.1
+tokenizers==0.21.1
+GitPython==3.1.44
+xxhash==3.5.0
+packaging==24.2
+numpy==1.23.4
+setproctitle==1.3.5
+llvmlite==0.43.0
+tiktoken==0.9.0
+mpmath==1.3.0
+nvidia-ml-py==12.570.86
+pydantic==2.11.0
+datasets==3.5.0
+librosa==0.11.0
+frechet_audio_distance==0.3.1
+sympy==1.13.1
+safehttpx==0.1.6
+Jinja2==3.1.6
+h11==0.14.0
+aiohttp==3.11.14
+diffusers==0.32.2
+tqdm==4.67.1
+filelock==3.13.1
+transformers==4.51.0.dev0
+scipy==1.10.1
+audioread==3.0.1
+sentencepiece==0.2.0
+pytz==2025.2
+tzdata==2025.2
+python-multipart==0.0.20
+urllib3==2.3.0
+pycryptodomex==3.22.0
+yarl==1.18.3
+pydub==0.25.1
+pycparser==2.22
+soundfile==0.13.1
+wcwidth==0.2.13
+groovy==0.1.2
+torch==2.5.1
+anyio==4.9.0
+wandb==0.19.8
+joblib==1.4.2
+fsspec==2024.12.0
+peft==0.15.1
+accelerate==1.5.2
+py-cpuinfo==9.0.0
+uvicorn==0.34.0
+orjson==3.10.16
+Brotli==1.0.9
+rich==13.9.4
+importlib_metadata==8.6.1
+ninja==1.11.1.4
+wheel==0.45.1
+MarkupSafe==3.0.2
+threadpoolctl==3.6.0
+gitdb==4.0.12
+mkl-service==2.4.0
+typing_extensions==4.12.2
+tomli==2.0.1
+zipp==3.19.2
+wheel==0.43.0
+jaraco.text==3.12.1
+packaging==24.2
+autocommand==2.2.2
+jaraco.functools==4.0.1
+jaraco.collections==5.1.0
+platformdirs==4.2.2
+more-itertools==10.3.0
+inflect==7.3.1
+jaraco.context==5.3.0
+typeguard==4.3.0
+backports.tarfile==1.2.0
+importlib_metadata==8.0.0

offline-run-20250328_203844-gc2qytpj/files/wandb-metadata.json

@@ -0,0 +1,105 @@
+{
+  "os": "Linux-5.15.0-1040-nvidia-x86_64-with-glibc2.35",
+  "python": "CPython 3.11.11",
+  "startedAt": "2025-03-28T12:38:44.660819Z",
+  "args": [
+    "--local_rank=0",
+    "--model_name_or_path",
+    "/aifs4su/yaodong/hantao/models/llava-v1.6-vicuna-13b-hf",
+    "--train_datasets",
+    "/aifs4su/yaodong/hantao/datasets/AA_preference_vicuna-13b_cosi_cut/merged/top1-10",
+    "--train_template",
+    "AA_TI2T_LLAVA",
+    "--train_split",
+    "train",
+    "--train_name",
+    "text-image-to-text",
+    "--output_dir",
+    "../outputs/test_13B",
+    "--save_total_limit",
+    "6",
+    "--train_batch_size",
+    "2",
+    "--epochs",
+    "3"
+  ],
+  "program": "-m align_anything.trainers.text_image_to_text.dpo",
+  "git": {
+    "remote": "git@github.com-hantao:PKU-Alignment/align-anything.git",
+    "commit": "106588f9802757a3283c1aff1f33ea9afd737f31"
+  },
+  "root": "../outputs/test_13B",
+  "host": "dgx-092",
+  "executable": "/aifs4su/yaodong/miniconda3/envs/hantao_llama/bin/python",
+  "cpu_count": 112,
+  "cpu_count_logical": 224,
+  "gpu": "NVIDIA H800",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1888556142592",
+      "used": "148592906240"
+    }
+  },
+  "memory": {
+    "total": "2164195454976"
+  },
+  "cpu": {
+    "count": 112,
+    "countLogical": 224
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA H800",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H800",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H800",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H800",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H800",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H800",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H800",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H800",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    }
+  ],
+  "slurm": {
+    "conf": "/cm/shared/apps/slurm/var/etc/slurm/slurm.conf"
+  },
+  "cudaVersion": "12.2"
+}

offline-run-20250328_203844-gc2qytpj/logs/debug-core.log

@@ -0,0 +1,7 @@
+{"time":"2025-03-28T20:38:44.475652335+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpe73e3keo/port-3871311.txt","pid":3871311,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false}
+{"time":"2025-03-28T20:38:44.476536664+08:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":3871311}
+{"time":"2025-03-28T20:38:44.476519981+08:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":37967,"Zone":""}}
+{"time":"2025-03-28T20:38:44.657114013+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:34698"}
+{"time":"2025-03-28T20:38:44.662194892+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"gc2qytpj","id":"127.0.0.1:34698"}
+{"time":"2025-03-28T20:38:44.906611127+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"gc2qytpj","id":"127.0.0.1:34698"}
+{"time":"2025-03-28T20:47:40.250303335+08:00","level":"INFO","msg":"received shutdown signal","signal":15}

offline-run-20250328_203844-gc2qytpj/logs/debug-internal.log

@@ -0,0 +1,7 @@
+{"time":"2025-03-28T20:38:44.692258573+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"../outputs/test_13B/wandb/offline-run-20250328_203844-gc2qytpj/logs/debug-core.log"}
+{"time":"2025-03-28T20:38:44.90653769+08:00","level":"INFO","msg":"created new stream","id":"gc2qytpj"}
+{"time":"2025-03-28T20:38:44.90660149+08:00","level":"INFO","msg":"stream: started","id":"gc2qytpj"}
+{"time":"2025-03-28T20:38:44.906615038+08:00","level":"INFO","msg":"handler: started","stream_id":"gc2qytpj"}
+{"time":"2025-03-28T20:38:44.906633311+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"gc2qytpj"}
+{"time":"2025-03-28T20:38:44.90663431+08:00","level":"INFO","msg":"sender: started","stream_id":"gc2qytpj"}
+{"time":"2025-03-28T20:38:44.910439284+08:00","level":"INFO","msg":"Starting system monitor"}

offline-run-20250328_203844-gc2qytpj/logs/debug.log

@@ -0,0 +1,26 @@
+2025-03-28 20:38:44,447 INFO    MainThread:3871311 [wandb_setup.py:_flush():67] Current SDK version is 0.19.8
+2025-03-28 20:38:44,447 INFO    MainThread:3871311 [wandb_setup.py:_flush():67] Configure stats pid to 3871311
+2025-03-28 20:38:44,447 INFO    MainThread:3871311 [wandb_setup.py:_flush():67] Loading settings from /home/yangyaodong/.config/wandb/settings
+2025-03-28 20:38:44,447 INFO    MainThread:3871311 [wandb_setup.py:_flush():67] Loading settings from /aifs4su/yaodong/hantao/align-anything/scripts/wandb/settings
+2025-03-28 20:38:44,447 INFO    MainThread:3871311 [wandb_setup.py:_flush():67] Loading settings from environment variables
+2025-03-28 20:38:44,447 INFO    MainThread:3871311 [wandb_init.py:setup_run_log_directory():647] Logging user logs to ../outputs/test_13B/wandb/offline-run-20250328_203844-gc2qytpj/logs/debug.log
+2025-03-28 20:38:44,447 INFO    MainThread:3871311 [wandb_init.py:setup_run_log_directory():648] Logging internal logs to ../outputs/test_13B/wandb/offline-run-20250328_203844-gc2qytpj/logs/debug-internal.log
+2025-03-28 20:38:44,447 INFO    MainThread:3871311 [wandb_init.py:init():761] calling init triggers
+2025-03-28 20:38:44,447 INFO    MainThread:3871311 [wandb_init.py:init():766] wandb.init called with sweep_config: {}
+config: {'train_cfgs': {'save_checkpoint': True, 'load_checkpoint': False, 'ds_cfgs': 'ds_z3_config.json', 'epochs': 3, 'seed': 42, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 1, 'gradient_accumulation_steps': 1, 'gradient_checkpointing': True, 'learning_rate': 1e-06, 'lr_scheduler_type': 'cosine', 'lr_warmup_ratio': 0.03, 'weight_decay': 0.0, 'adam_betas': [0.9, 0.95], 'bf16': True, 'fp16': False, 'eval_strategy': 'epoch', 'eval_interval': 10, 'regularization': 0.001, 'scale_coeff': 0.1, 'freeze_mm_proj': False, 'freeze_vision_tower': True, 'freeze_language_model': False}, 'data_cfgs': {'train_datasets': '/aifs4su/yaodong/hantao/datasets/AA_preference_vicuna-13b_cosi_cut/merged/top1-10', 'train_template': 'AA_TI2T_LLAVA', 'train_size': {}, 'train_split': 'train', 'train_name': 'text-image-to-text', 'train_data_files': {}, 'train_optional_args': [], 'eval_datasets': {}, 'eval_template': {}, 'eval_size': {}, 'eval_split': {}, 'eval_subset': {}, 'eval_data_files': {}, 'eval_optional_args': []}, 'logger_cfgs': {'log_type': 'wandb', 'log_project': 'align-anything', 'log_run_name': 'dpo', 'output_dir': '../outputs/test_13B', 'cache_dir': {}, 'save_total_limit': 6}, 'model_cfgs': {'model_name_or_path': '/aifs4su/yaodong/hantao/models/llava-v1.6-vicuna-13b-hf', 'trust_remote_code': True, 'model_max_length': 2048}, 'special_tokens': {}, '_wandb': {}}
+2025-03-28 20:38:44,447 INFO    MainThread:3871311 [wandb_init.py:init():784] starting backend
+2025-03-28 20:38:44,657 INFO    MainThread:3871311 [wandb_init.py:init():788] sending inform_init request
+2025-03-28 20:38:44,660 INFO    MainThread:3871311 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-03-28 20:38:44,660 INFO    MainThread:3871311 [wandb_init.py:init():798] backend started and connected
+2025-03-28 20:38:44,662 INFO    MainThread:3871311 [wandb_init.py:init():891] updated telemetry
+2025-03-28 20:38:44,672 INFO    MainThread:3871311 [wandb_init.py:init():915] communicating run to backend with 90.0 second timeout
+2025-03-28 20:38:44,908 INFO    MainThread:3871311 [wandb_init.py:init():990] starting run threads in backend
+2025-03-28 20:38:45,071 INFO    MainThread:3871311 [wandb_run.py:_console_start():2375] atexit reg
+2025-03-28 20:38:45,071 INFO    MainThread:3871311 [wandb_run.py:_redirect():2227] redirect: wrap_raw
+2025-03-28 20:38:45,071 INFO    MainThread:3871311 [wandb_run.py:_redirect():2292] Wrapping output streams.
+2025-03-28 20:38:45,071 INFO    MainThread:3871311 [wandb_run.py:_redirect():2315] Redirects installed.
+2025-03-28 20:38:45,074 INFO    MainThread:3871311 [wandb_init.py:init():1032] run started, returning control to user process
+2025-03-28 20:47:40,230 INFO    MainThread:3871311 [wandb_run.py:_finish():2112] finishing run align-anything/gc2qytpj
+2025-03-28 20:47:40,230 INFO    MainThread:3871311 [wandb_run.py:_atexit_cleanup():2340] got exitcode: 0
+2025-03-28 20:47:40,230 INFO    MainThread:3871311 [wandb_run.py:_restore():2322] restore
+2025-03-28 20:47:40,230 INFO    MainThread:3871311 [wandb_run.py:_restore():2328] restore done

offline-run-20250328_203844-gc2qytpj/run-gc2qytpj.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8aa9b355d934125304017d0d212405953c1aba2e8d426c9e82a0b3097e7e3c89
+size 393216