Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+run-20250329_010934-3x35hjks/run-3x35hjks.wandb filter=lfs diff=lfs merge=lfs -text
+run-20250329_012205-co1ecmky/run-co1ecmky.wandb filter=lfs diff=lfs merge=lfs -text

debug-internal.log

@@ -0,0 +1,7 @@
+{"time":"2025-03-29T01:22:05.252520599+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"../outputs/test_7B/wandb/run-20250329_012205-co1ecmky/logs/debug-core.log"}
+{"time":"2025-03-29T01:22:05.467254306+08:00","level":"INFO","msg":"created new stream","id":"co1ecmky"}
+{"time":"2025-03-29T01:22:05.467309548+08:00","level":"INFO","msg":"stream: started","id":"co1ecmky"}
+{"time":"2025-03-29T01:22:05.467331707+08:00","level":"INFO","msg":"handler: started","stream_id":"co1ecmky"}
+{"time":"2025-03-29T01:22:05.467333162+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"co1ecmky"}
+{"time":"2025-03-29T01:22:05.467336174+08:00","level":"INFO","msg":"sender: started","stream_id":"co1ecmky"}
+{"time":"2025-03-29T01:22:05.772490021+08:00","level":"INFO","msg":"Starting system monitor"}

debug.log

@@ -0,0 +1,24 @@
+2025-03-29 01:22:05,246 INFO    MainThread:104999 [wandb_setup.py:_flush():67] Current SDK version is 0.19.8
+2025-03-29 01:22:05,246 INFO    MainThread:104999 [wandb_setup.py:_flush():67] Configure stats pid to 104999
+2025-03-29 01:22:05,246 INFO    MainThread:104999 [wandb_setup.py:_flush():67] Loading settings from /home/yangyaodong/.config/wandb/settings
+2025-03-29 01:22:05,246 INFO    MainThread:104999 [wandb_setup.py:_flush():67] Loading settings from /aifs4su/yaodong/hantao/align-anything/scripts/wandb/settings
+2025-03-29 01:22:05,246 INFO    MainThread:104999 [wandb_setup.py:_flush():67] Loading settings from environment variables
+2025-03-29 01:22:05,246 INFO    MainThread:104999 [wandb_init.py:setup_run_log_directory():647] Logging user logs to ../outputs/test_7B/wandb/run-20250329_012205-co1ecmky/logs/debug.log
+2025-03-29 01:22:05,246 INFO    MainThread:104999 [wandb_init.py:setup_run_log_directory():648] Logging internal logs to ../outputs/test_7B/wandb/run-20250329_012205-co1ecmky/logs/debug-internal.log
+2025-03-29 01:22:05,246 INFO    MainThread:104999 [wandb_init.py:init():761] calling init triggers
+2025-03-29 01:22:05,246 INFO    MainThread:104999 [wandb_init.py:init():766] wandb.init called with sweep_config: {}
+config: {'train_cfgs': {'save_checkpoint': False, 'load_checkpoint': False, 'ds_cfgs': 'ds_z3_config.json', 'epochs': 3, 'seed': 42, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 1, 'gradient_accumulation_steps': 16, 'gradient_checkpointing': True, 'learning_rate': 2e-05, 'lr_scheduler_type': 'cosine', 'lr_warmup_ratio': 0.03, 'weight_decay': 0.0, 'adam_betas': [0.9, 0.95], 'adam_epsilon': 1e-08, 'bf16': True, 'fp16': False, 'eval_strategy': 'epoch', 'eval_interval': 10, 'freeze_mm_proj': False, 'freeze_vision_tower': True, 'freeze_language_model': False, 'max_grad_norm': 1.0}, 'data_cfgs': {'load_multi_datasets': False, 'train_datasets': '/aifs4su/yaodong/hantao/datasets/MMInstruct-GPT4V_mistral-7b_cosi_cut/merged/top1-10', 'train_template': 'MM_TI2T_LLAVA', 'train_size': {}, 'train_split': 'train', 'train_name': 'text-image-to-text', 'train_data_files': {}, 'train_optional_args': [], 'eval_datasets': {}, 'eval_template': {}, 'eval_name': {}, 'eval_size': {}, 'eval_split': {}, 'eval_subset': {}, 'eval_data_files': {}, 'eval_optional_args': []}, 'logger_cfgs': {'log_type': 'wandb', 'log_project': 'align-anything', 'log_run_name': 'sft', 'output_dir': '../outputs/test_7B', 'cache_dir': {}, 'save_total_limit': 6}, 'model_cfgs': {'model_name_or_path': '/aifs4su/yaodong/hantao/models/llava-v1.6-mistral-7b-hf', 'trust_remote_code': True, 'model_max_length': 2048}, 'special_tokens': {}, '_wandb': {}}
+2025-03-29 01:22:05,246 INFO    MainThread:104999 [wandb_init.py:init():784] starting backend
+2025-03-29 01:22:05,246 INFO    MainThread:104999 [wandb_init.py:init():788] sending inform_init request
+2025-03-29 01:22:05,249 INFO    MainThread:104999 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-03-29 01:22:05,249 INFO    MainThread:104999 [wandb_init.py:init():798] backend started and connected
+2025-03-29 01:22:05,251 INFO    MainThread:104999 [wandb_init.py:init():891] updated telemetry
+2025-03-29 01:22:05,262 INFO    MainThread:104999 [wandb_init.py:init():915] communicating run to backend with 90.0 second timeout
+2025-03-29 01:22:05,770 INFO    MainThread:104999 [wandb_init.py:init():990] starting run threads in backend
+2025-03-29 01:22:05,989 INFO    MainThread:104999 [wandb_run.py:_console_start():2375] atexit reg
+2025-03-29 01:22:05,990 INFO    MainThread:104999 [wandb_run.py:_redirect():2227] redirect: wrap_raw
+2025-03-29 01:22:05,990 INFO    MainThread:104999 [wandb_run.py:_redirect():2292] Wrapping output streams.
+2025-03-29 01:22:05,990 INFO    MainThread:104999 [wandb_run.py:_redirect():2315] Redirects installed.
+2025-03-29 01:22:05,992 INFO    MainThread:104999 [wandb_init.py:init():1032] run started, returning control to user process
+2025-03-29 01:26:51,278 INFO    MainThread:104999 [wandb_run.py:_finish():2112] finishing run htlou/align-anything/co1ecmky
+2025-03-29 01:26:51,278 INFO    MainThread:104999 [wandb_run.py:_atexit_cleanup():2340] got exitcode: 0

run-20250329_003552-1mlhe6om/files/config.yaml

@@ -0,0 +1,95 @@
+_wandb:
+    value:
+        cli_version: 0.19.8
+        m: []
+        python_version: 3.11.11
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 41
+                - 49
+                - 51
+                - 53
+                - 55
+                - 63
+                - 71
+                - 83
+                - 98
+                - 105
+            "2":
+                - 1
+                - 5
+                - 11
+                - 41
+                - 49
+                - 51
+                - 53
+                - 55
+                - 63
+                - 71
+                - 83
+                - 98
+                - 105
+            "3":
+                - 2
+                - 13
+                - 16
+                - 23
+                - 55
+            "4": 3.11.11
+            "5": 0.19.8
+            "6": 4.51.0.dev0
+            "8":
+                - 5
+            "12": 0.19.8
+            "13": linux-x86_64
+data_cfgs:
+    value:
+        eval_optional_args: []
+        load_multi_datasets: false
+        train_datasets: /aifs4su/yaodong/hantao/datasets/MMInstruct-GPT4V_mistral-7b_cosi_cut/merged/top1-10
+        train_name: text-image-to-text
+        train_optional_args: []
+        train_split: train
+        train_template: MM_TI2T_LLAVA
+logger_cfgs:
+    value:
+        log_project: align-anything
+        log_run_name: sft
+        log_type: wandb
+        output_dir: ../outputs/test_7B
+        save_total_limit: 6
+model_cfgs:
+    value:
+        model_max_length: 2048
+        model_name_or_path: /aifs4su/yaodong/hantao/models/llava-v1.6-mistral-7b-hf
+        trust_remote_code: true
+train_cfgs:
+    value:
+        adam_betas:
+            - 0.9
+            - 0.95
+        adam_epsilon: 1e-08
+        bf16: true
+        ds_cfgs: ds_z3_config.json
+        epochs: 3
+        eval_interval: 10
+        eval_strategy: epoch
+        fp16: false
+        freeze_language_model: false
+        freeze_mm_proj: false
+        freeze_vision_tower: true
+        gradient_accumulation_steps: 16
+        gradient_checkpointing: true
+        learning_rate: 2e-05
+        load_checkpoint: false
+        lr_scheduler_type: cosine
+        lr_warmup_ratio: 0.03
+        max_grad_norm: 1
+        per_device_eval_batch_size: 1
+        per_device_train_batch_size: 1
+        save_checkpoint: true
+        seed: 42
+        weight_decay: 0

run-20250329_003552-1mlhe6om/files/output.log

@@ -0,0 +1,21 @@
+***** Running training *****
+Resuming from checkpoint 3/3 epoch :   0%|                                                                                                                                                                                                                                                                    | 0/7326 [00:00<?, ?it/s]
+Current epoch:  0
+Current epoch:  1
+Current epoch:  2
+Saving model to "../outputs/test_7B/slice_end" ...
+Saving 16-bit model...
+[2025-03-29 00:36:00,887] [INFO] [logging.py:107:log_dist] [Rank 0] [Torch] Checkpoint global_step0 is about to be saved!
+[2025-03-29 00:36:00,887] [INFO] [engine.py:3831:save_16bit_model] Saving model weights to ../outputs/test_7B/slice_end/pytorch_model.bin, tag: global_step0
+[2025-03-29 00:36:00,888] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/test_7B/slice_end/pytorch_model.bin...
+[2025-03-29 00:36:14,412] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/test_7B/slice_end/pytorch_model.bin.
+[2025-03-29 00:36:14,412] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step0 is ready now!
+[2025-03-29 00:36:15,002] [INFO] [logging.py:107:log_dist] [Rank 0] [Torch] Checkpoint global_step0 is about to be saved!
+[2025-03-29 00:36:15,045] [INFO] [logging.py:107:log_dist] [Rank 0] Saving model checkpoint: ../outputs/test_7B/slice_end/global_step0/zero_pp_rank_0_mp_rank_00_model_states.pt
+[2025-03-29 00:36:15,045] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/test_7B/slice_end/global_step0/zero_pp_rank_0_mp_rank_00_model_states.pt...
+[2025-03-29 00:36:15,195] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/test_7B/slice_end/global_step0/zero_pp_rank_0_mp_rank_00_model_states.pt.
+[2025-03-29 00:36:20,367] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/test_7B/slice_end/global_step0/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt...
+[2025-03-29 00:36:26,225] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/test_7B/slice_end/global_step0/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt.
+[2025-03-29 00:36:26,228] [INFO] [engine.py:3672:_save_zero_checkpoint] zero checkpoint saved ../outputs/test_7B/slice_end/global_step0/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
+[2025-03-29 00:36:26,608] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step0 is ready now!
+Model saved!

run-20250329_003552-1mlhe6om/files/requirements.txt

@@ -0,0 +1,167 @@
+maskrcnn_benchmark==0.0.0
+webdataset==0.2.111
+websockets==15.0.1
+typer==0.15.2
+blobfile==3.0.0
+pooch==1.8.2
+python-dateutil==2.9.0.post0
+gmpy2==2.2.1
+httpcore==1.0.7
+charset-normalizer==3.3.2
+torchlibrosa==0.1.0
+multiprocess==0.70.16
+Werkzeug==3.1.3
+aiofiles==23.2.1
+six==1.17.0
+typing_extensions==4.12.2
+psutil==7.0.0
+frozenlist==1.5.0
+einops==0.8.1
+flash_attn==2.7.4.post1
+PySocks==1.7.1
+regex==2024.11.6
+markdown-it-py==3.0.0
+ruff==0.11.2
+docker-pycreds==0.4.0
+protobuf==5.29.4
+resampy==0.4.3
+aiohappyeyeballs==2.6.1
+httpx==0.28.1
+encodec==0.1.1
+ffmpy==0.5.0
+mkl_random==1.2.8
+soxr==0.5.0.post1
+absl-py==2.2.1
+networkx==3.4.2
+h5py==3.13.0
+hjson==3.1.0
+tensorboard==2.19.0
+aiosignal==1.3.2
+pip==25.0
+triton==3.1.0
+zipp==3.21.0
+ftfy==6.3.1
+attrs==25.3.0
+requests==2.32.3
+progressbar==2.5
+sniffio==1.3.1
+lxml==5.3.1
+starlette==0.46.1
+Markdown==3.7
+mdurl==0.1.2
+torchaudio==2.5.1
+safetensors==0.5.3
+opencv-python==4.6.0.66
+torchvision==0.20.1
+shellingham==1.5.4
+gradio==5.23.1
+timm==1.0.15
+multidict==6.2.0
+semantic-version==2.10.0
+numba==0.60.0
+gradio_client==1.8.0
+pydantic_core==2.33.0
+dill==0.3.8
+msgpack==1.1.0
+sentry-sdk==2.24.1
+grpcio==1.71.0
+cffi==1.17.1
+PyYAML==6.0.2
+tensorboard-data-server==0.7.2
+fastapi==0.115.12
+lazy_loader==0.4
+mkl_fft==1.3.11
+annotated-types==0.7.0
+scikit-learn==1.6.1
+wget==3.2
+setuptools==75.8.0
+certifi==2025.1.31
+click==8.1.8
+laion_clap==1.1.5
+Pygments==2.19.1
+tomlkit==0.13.2
+idna==3.7
+propcache==0.3.1
+platformdirs==4.3.7
+align-anything==0.0.1.dev0
+deepspeed==0.16.5
+smmap==5.0.2
+pillow==11.1.0
+typing-inspection==0.4.0
+braceexpand==0.1.7
+decorator==5.2.1
+pandas==2.2.3
+huggingface-hub==0.29.3
+pyarrow==19.0.1
+tokenizers==0.21.1
+GitPython==3.1.44
+xxhash==3.5.0
+packaging==24.2
+numpy==1.23.4
+setproctitle==1.3.5
+llvmlite==0.43.0
+tiktoken==0.9.0
+mpmath==1.3.0
+nvidia-ml-py==12.570.86
+pydantic==2.11.0
+datasets==3.5.0
+librosa==0.11.0
+frechet_audio_distance==0.3.1
+sympy==1.13.1
+safehttpx==0.1.6
+Jinja2==3.1.6
+h11==0.14.0
+aiohttp==3.11.14
+diffusers==0.32.2
+tqdm==4.67.1
+filelock==3.13.1
+transformers==4.51.0.dev0
+scipy==1.10.1
+audioread==3.0.1
+sentencepiece==0.2.0
+pytz==2025.2
+tzdata==2025.2
+python-multipart==0.0.20
+urllib3==2.3.0
+pycryptodomex==3.22.0
+yarl==1.18.3
+pydub==0.25.1
+pycparser==2.22
+soundfile==0.13.1
+wcwidth==0.2.13
+groovy==0.1.2
+torch==2.5.1
+anyio==4.9.0
+wandb==0.19.8
+joblib==1.4.2
+fsspec==2024.12.0
+peft==0.15.1
+accelerate==1.5.2
+py-cpuinfo==9.0.0
+uvicorn==0.34.0
+orjson==3.10.16
+Brotli==1.0.9
+rich==13.9.4
+importlib_metadata==8.6.1
+ninja==1.11.1.4
+wheel==0.45.1
+MarkupSafe==3.0.2
+threadpoolctl==3.6.0
+gitdb==4.0.12
+mkl-service==2.4.0
+typing_extensions==4.12.2
+tomli==2.0.1
+zipp==3.19.2
+wheel==0.43.0
+jaraco.text==3.12.1
+packaging==24.2
+autocommand==2.2.2
+jaraco.functools==4.0.1
+jaraco.collections==5.1.0
+platformdirs==4.2.2
+more-itertools==10.3.0
+inflect==7.3.1
+jaraco.context==5.3.0
+typeguard==4.3.0
+backports.tarfile==1.2.0
+importlib_metadata==8.0.0

run-20250329_003552-1mlhe6om/files/wandb-metadata.json

@@ -0,0 +1,106 @@
+{
+  "os": "Linux-5.15.0-1040-nvidia-x86_64-with-glibc2.35",
+  "python": "CPython 3.11.11",
+  "startedAt": "2025-03-28T16:35:52.296406Z",
+  "args": [
+    "--local_rank=0",
+    "--model_name_or_path",
+    "/aifs4su/yaodong/hantao/models/llava-v1.6-mistral-7b-hf",
+    "--train_datasets",
+    "/aifs4su/yaodong/hantao/datasets/MMInstruct-GPT4V_mistral-7b_cosi_cut/merged/top1-10",
+    "--train_template",
+    "MM_TI2T_LLAVA",
+    "--train_split",
+    "train",
+    "--train_name",
+    "text-image-to-text",
+    "--output_dir",
+    "../outputs/test_7B",
+    "--save_total_limit",
+    "6",
+    "--train_batch_size",
+    "8",
+    "--epochs",
+    "3"
+  ],
+  "program": "-m align_anything.trainers.text_image_to_text.sft",
+  "git": {
+    "remote": "git@github.com-hantao:PKU-Alignment/align-anything.git",
+    "commit": "106588f9802757a3283c1aff1f33ea9afd737f31"
+  },
+  "email": "2200017789@stu.pku.edu.cn",
+  "root": "../outputs/test_7B",
+  "host": "dgx-092",
+  "executable": "/aifs4su/yaodong/miniconda3/envs/hantao_llama/bin/python",
+  "cpu_count": 112,
+  "cpu_count_logical": 224,
+  "gpu": "NVIDIA H800",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1888556142592",
+      "used": "148607098880"
+    }
+  },
+  "memory": {
+    "total": "2164195454976"
+  },
+  "cpu": {
+    "count": 112,
+    "countLogical": 224
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA H800",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H800",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H800",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H800",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H800",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H800",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H800",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H800",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    }
+  ],
+  "slurm": {
+    "conf": "/cm/shared/apps/slurm/var/etc/slurm/slurm.conf"
+  },
+  "cudaVersion": "12.2"
+}

run-20250329_003552-1mlhe6om/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":34}}

run-20250329_003552-1mlhe6om/logs/debug-core.log

@@ -0,0 +1,13 @@
+{"time":"2025-03-29T00:35:51.700332688+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp4xrq9hq1/port-9551.txt","pid":9551,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false}
+{"time":"2025-03-29T00:35:51.701486575+08:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":9551}
+{"time":"2025-03-29T00:35:51.701420457+08:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":41983,"Zone":""}}
+{"time":"2025-03-29T00:35:51.880332094+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:61422"}
+{"time":"2025-03-29T00:35:52.297979257+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"1mlhe6om","id":"127.0.0.1:61422"}
+{"time":"2025-03-29T00:35:52.514464455+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"1mlhe6om","id":"127.0.0.1:61422"}
+{"time":"2025-03-29T00:36:28.014384472+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:61422"}
+{"time":"2025-03-29T00:36:28.014433444+08:00","level":"INFO","msg":"connection: closing","id":"127.0.0.1:61422"}
+{"time":"2025-03-29T00:36:28.014448354+08:00","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-03-29T00:36:28.014489675+08:00","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:61422"}
+{"time":"2025-03-29T00:36:28.023136355+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:61422"}
+{"time":"2025-03-29T00:36:28.023152325+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:61422"}
+{"time":"2025-03-29T00:36:28.023157944+08:00","level":"INFO","msg":"server is closed"}

run-20250329_003552-1mlhe6om/logs/debug-internal.log

@@ -0,0 +1,16 @@
+{"time":"2025-03-29T00:35:52.299334516+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"../outputs/test_7B/wandb/run-20250329_003552-1mlhe6om/logs/debug-core.log"}
+{"time":"2025-03-29T00:35:52.5143997+08:00","level":"INFO","msg":"created new stream","id":"1mlhe6om"}
+{"time":"2025-03-29T00:35:52.514457632+08:00","level":"INFO","msg":"stream: started","id":"1mlhe6om"}
+{"time":"2025-03-29T00:35:52.514478469+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"1mlhe6om"}
+{"time":"2025-03-29T00:35:52.514486118+08:00","level":"INFO","msg":"handler: started","stream_id":"1mlhe6om"}
+{"time":"2025-03-29T00:35:52.514497089+08:00","level":"INFO","msg":"sender: started","stream_id":"1mlhe6om"}
+{"time":"2025-03-29T00:35:52.799983877+08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2025-03-29T00:36:26.630995684+08:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2025-03-29T00:36:26.631709662+08:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2025-03-29T00:36:27.77584189+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-03-29T00:36:28.01449504+08:00","level":"INFO","msg":"stream: closing","id":"1mlhe6om"}
+{"time":"2025-03-29T00:36:28.01455515+08:00","level":"INFO","msg":"handler: closed","stream_id":"1mlhe6om"}
+{"time":"2025-03-29T00:36:28.014530377+08:00","level":"WARN","msg":"sender: received Exit record more than once, ignoring"}
+{"time":"2025-03-29T00:36:28.014604384+08:00","level":"INFO","msg":"sender: closed","stream_id":"1mlhe6om"}
+{"time":"2025-03-29T00:36:28.014567177+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"1mlhe6om"}
+{"time":"2025-03-29T00:36:28.02303065+08:00","level":"INFO","msg":"stream: closed","id":"1mlhe6om"}

run-20250329_003552-1mlhe6om/logs/debug.log

@@ -0,0 +1,47 @@
+2025-03-29 00:35:52,291 INFO    MainThread:9551 [wandb_setup.py:_flush():67] Current SDK version is 0.19.8
+2025-03-29 00:35:52,292 INFO    MainThread:9551 [wandb_setup.py:_flush():67] Configure stats pid to 9551
+2025-03-29 00:35:52,292 INFO    MainThread:9551 [wandb_setup.py:_flush():67] Loading settings from /home/yangyaodong/.config/wandb/settings
+2025-03-29 00:35:52,292 INFO    MainThread:9551 [wandb_setup.py:_flush():67] Loading settings from /aifs4su/yaodong/hantao/align-anything/scripts/wandb/settings
+2025-03-29 00:35:52,292 INFO    MainThread:9551 [wandb_setup.py:_flush():67] Loading settings from environment variables
+2025-03-29 00:35:52,292 INFO    MainThread:9551 [wandb_init.py:setup_run_log_directory():647] Logging user logs to ../outputs/test_7B/wandb/run-20250329_003552-1mlhe6om/logs/debug.log
+2025-03-29 00:35:52,292 INFO    MainThread:9551 [wandb_init.py:setup_run_log_directory():648] Logging internal logs to ../outputs/test_7B/wandb/run-20250329_003552-1mlhe6om/logs/debug-internal.log
+2025-03-29 00:35:52,292 INFO    MainThread:9551 [wandb_init.py:init():761] calling init triggers
+2025-03-29 00:35:52,292 INFO    MainThread:9551 [wandb_init.py:init():766] wandb.init called with sweep_config: {}
+config: {'train_cfgs': {'save_checkpoint': True, 'load_checkpoint': False, 'ds_cfgs': 'ds_z3_config.json', 'epochs': 3, 'seed': 42, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 1, 'gradient_accumulation_steps': 16, 'gradient_checkpointing': True, 'learning_rate': 2e-05, 'lr_scheduler_type': 'cosine', 'lr_warmup_ratio': 0.03, 'weight_decay': 0.0, 'adam_betas': [0.9, 0.95], 'adam_epsilon': 1e-08, 'bf16': True, 'fp16': False, 'eval_strategy': 'epoch', 'eval_interval': 10, 'freeze_mm_proj': False, 'freeze_vision_tower': True, 'freeze_language_model': False, 'max_grad_norm': 1.0}, 'data_cfgs': {'load_multi_datasets': False, 'train_datasets': '/aifs4su/yaodong/hantao/datasets/MMInstruct-GPT4V_mistral-7b_cosi_cut/merged/top1-10', 'train_template': 'MM_TI2T_LLAVA', 'train_size': {}, 'train_split': 'train', 'train_name': 'text-image-to-text', 'train_data_files': {}, 'train_optional_args': [], 'eval_datasets': {}, 'eval_template': {}, 'eval_name': {}, 'eval_size': {}, 'eval_split': {}, 'eval_subset': {}, 'eval_data_files': {}, 'eval_optional_args': []}, 'logger_cfgs': {'log_type': 'wandb', 'log_project': 'align-anything', 'log_run_name': 'sft', 'output_dir': '../outputs/test_7B', 'cache_dir': {}, 'save_total_limit': 6}, 'model_cfgs': {'model_name_or_path': '/aifs4su/yaodong/hantao/models/llava-v1.6-mistral-7b-hf', 'trust_remote_code': True, 'model_max_length': 2048}, 'special_tokens': {}, '_wandb': {}}
+2025-03-29 00:35:52,292 INFO    MainThread:9551 [wandb_init.py:init():784] starting backend
+2025-03-29 00:35:52,292 INFO    MainThread:9551 [wandb_init.py:init():788] sending inform_init request
+2025-03-29 00:35:52,296 INFO    MainThread:9551 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-03-29 00:35:52,296 INFO    MainThread:9551 [wandb_init.py:init():798] backend started and connected
+2025-03-29 00:35:52,297 INFO    MainThread:9551 [wandb_init.py:init():891] updated telemetry
+2025-03-29 00:35:52,308 INFO    MainThread:9551 [wandb_init.py:init():915] communicating run to backend with 90.0 second timeout
+2025-03-29 00:35:52,796 INFO    MainThread:9551 [wandb_init.py:init():990] starting run threads in backend
+2025-03-29 00:35:52,947 INFO    MainThread:9551 [wandb_run.py:_console_start():2375] atexit reg
+2025-03-29 00:35:52,947 INFO    MainThread:9551 [wandb_run.py:_redirect():2227] redirect: wrap_raw
+2025-03-29 00:35:52,947 INFO    MainThread:9551 [wandb_run.py:_redirect():2292] Wrapping output streams.
+2025-03-29 00:35:52,947 INFO    MainThread:9551 [wandb_run.py:_redirect():2315] Redirects installed.
+2025-03-29 00:35:52,949 INFO    MainThread:9551 [wandb_init.py:init():1032] run started, returning control to user process
+2025-03-29 00:36:26,629 INFO    MainThread:9551 [wandb_run.py:_finish():2112] finishing run htlou/align-anything/1mlhe6om
+2025-03-29 00:36:26,630 INFO    MainThread:9551 [wandb_run.py:_atexit_cleanup():2340] got exitcode: 0
+2025-03-29 00:36:26,630 INFO    MainThread:9551 [wandb_run.py:_restore():2322] restore
+2025-03-29 00:36:26,630 INFO    MainThread:9551 [wandb_run.py:_restore():2328] restore done
+2025-03-29 00:36:27,631 INFO    MainThread:9551 [wandb_run.py:_restore():2322] restore
+2025-03-29 00:36:27,631 INFO    MainThread:9551 [wandb_run.py:_restore():2328] restore done
+2025-03-29 00:36:27,631 ERROR   MainThread:9551 [wandb_run.py:_atexit_cleanup():2361] Problem finishing run
+Traceback (most recent call last):
+  File "/aifs4su/yaodong/miniconda3/envs/hantao_llama/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2352, in _atexit_cleanup
+    self._on_finish()
+  File "/aifs4su/yaodong/miniconda3/envs/hantao_llama/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2609, in _on_finish
+    wait_with_progress(
+  File "/aifs4su/yaodong/miniconda3/envs/hantao_llama/lib/python3.11/site-packages/wandb/sdk/mailbox/wait_with_progress.py", line 24, in wait_with_progress
+    return wait_all_with_progress(
+           ^^^^^^^^^^^^^^^^^^^^^^^
+  File "/aifs4su/yaodong/miniconda3/envs/hantao_llama/lib/python3.11/site-packages/wandb/sdk/mailbox/wait_with_progress.py", line 87, in wait_all_with_progress
+    return asyncio_compat.run(progress_loop_with_timeout)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/aifs4su/yaodong/miniconda3/envs/hantao_llama/lib/python3.11/site-packages/wandb/sdk/lib/asyncio_compat.py", line 27, in run
+    future = executor.submit(runner.run, fn)
+             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/aifs4su/yaodong/miniconda3/envs/hantao_llama/lib/python3.11/concurrent/futures/thread.py", line 169, in submit
+    raise RuntimeError('cannot schedule new futures after '
+RuntimeError: cannot schedule new futures after interpreter shutdown
+2025-03-29 00:36:28,013 INFO    MsgRouterThr:9551 [mailbox.py:close():129] Closing mailbox, abandoning 1 handles.

run-20250329_003923-yubb37lj/files/output.log

@@ -0,0 +1,16 @@
+***** Running training *****
+Resuming from checkpoint 3/3 epoch :   0%|                                                                                                                                                                                                                                                                    | 0/7326 [00:00<?, ?it/s]
+Current epoch:  0
+Current epoch:  1
+Current epoch:  2
+Saving model to "../outputs/test_7B/slice_end" ...
+Saving 16-bit model...
+[2025-03-29 00:39:32,108] [INFO] [logging.py:107:log_dist] [Rank 0] [Torch] Checkpoint global_step0 is about to be saved!
+[2025-03-29 00:39:32,108] [INFO] [engine.py:3831:save_16bit_model] Saving model weights to ../outputs/test_7B/slice_end/pytorch_model.bin, tag: global_step0
+[2025-03-29 00:39:32,109] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/test_7B/slice_end/pytorch_model.bin...
+[2025-03-29 00:39:46,748] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/test_7B/slice_end/pytorch_model.bin.
+[2025-03-29 00:39:46,748] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step0 is ready now!
+[2025-03-29 00:39:47,316] [INFO] [logging.py:107:log_dist] [Rank 0] [Torch] Checkpoint global_step0 is about to be saved!
+[2025-03-29 00:39:47,344] [INFO] [logging.py:107:log_dist] [Rank 0] Saving model checkpoint: ../outputs/test_7B/slice_end/global_step0/zero_pp_rank_0_mp_rank_00_model_states.pt
+[2025-03-29 00:39:47,345] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/test_7B/slice_end/global_step0/zero_pp_rank_0_mp_rank_00_model_states.pt...
+[2025-03-29 00:39:47,453] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/test_7B/slice_end/global_step0/zero_pp_rank_0_mp_rank_00_model_states.pt.

run-20250329_003923-yubb37lj/files/requirements.txt

@@ -0,0 +1,167 @@
+maskrcnn_benchmark==0.0.0
+webdataset==0.2.111
+websockets==15.0.1
+typer==0.15.2
+blobfile==3.0.0
+pooch==1.8.2
+python-dateutil==2.9.0.post0
+gmpy2==2.2.1
+httpcore==1.0.7
+charset-normalizer==3.3.2
+torchlibrosa==0.1.0
+multiprocess==0.70.16
+Werkzeug==3.1.3
+aiofiles==23.2.1
+six==1.17.0
+typing_extensions==4.12.2
+psutil==7.0.0
+frozenlist==1.5.0
+einops==0.8.1
+flash_attn==2.7.4.post1
+PySocks==1.7.1
+regex==2024.11.6
+markdown-it-py==3.0.0
+ruff==0.11.2
+docker-pycreds==0.4.0
+protobuf==5.29.4
+resampy==0.4.3
+aiohappyeyeballs==2.6.1
+httpx==0.28.1
+encodec==0.1.1
+ffmpy==0.5.0
+mkl_random==1.2.8
+soxr==0.5.0.post1
+absl-py==2.2.1
+networkx==3.4.2
+h5py==3.13.0
+hjson==3.1.0
+tensorboard==2.19.0
+aiosignal==1.3.2
+pip==25.0
+triton==3.1.0
+zipp==3.21.0
+ftfy==6.3.1
+attrs==25.3.0
+requests==2.32.3
+progressbar==2.5
+sniffio==1.3.1
+lxml==5.3.1
+starlette==0.46.1
+Markdown==3.7
+mdurl==0.1.2
+torchaudio==2.5.1
+safetensors==0.5.3
+opencv-python==4.6.0.66
+torchvision==0.20.1
+shellingham==1.5.4
+gradio==5.23.1
+timm==1.0.15
+multidict==6.2.0
+semantic-version==2.10.0
+numba==0.60.0
+gradio_client==1.8.0
+pydantic_core==2.33.0
+dill==0.3.8
+msgpack==1.1.0
+sentry-sdk==2.24.1
+grpcio==1.71.0
+cffi==1.17.1
+PyYAML==6.0.2
+tensorboard-data-server==0.7.2
+fastapi==0.115.12
+lazy_loader==0.4
+mkl_fft==1.3.11
+annotated-types==0.7.0
+scikit-learn==1.6.1
+wget==3.2
+setuptools==75.8.0
+certifi==2025.1.31
+click==8.1.8
+laion_clap==1.1.5
+Pygments==2.19.1
+tomlkit==0.13.2
+idna==3.7
+propcache==0.3.1
+platformdirs==4.3.7
+align-anything==0.0.1.dev0
+deepspeed==0.16.5
+smmap==5.0.2
+pillow==11.1.0
+typing-inspection==0.4.0
+braceexpand==0.1.7
+decorator==5.2.1
+pandas==2.2.3
+huggingface-hub==0.29.3
+pyarrow==19.0.1
+tokenizers==0.21.1
+GitPython==3.1.44
+xxhash==3.5.0
+packaging==24.2
+numpy==1.23.4
+setproctitle==1.3.5
+llvmlite==0.43.0
+tiktoken==0.9.0
+mpmath==1.3.0
+nvidia-ml-py==12.570.86
+pydantic==2.11.0
+datasets==3.5.0
+librosa==0.11.0
+frechet_audio_distance==0.3.1
+sympy==1.13.1
+safehttpx==0.1.6
+Jinja2==3.1.6
+h11==0.14.0
+aiohttp==3.11.14
+diffusers==0.32.2
+tqdm==4.67.1
+filelock==3.13.1
+transformers==4.51.0.dev0
+scipy==1.10.1
+audioread==3.0.1
+sentencepiece==0.2.0
+pytz==2025.2
+tzdata==2025.2
+python-multipart==0.0.20
+urllib3==2.3.0
+pycryptodomex==3.22.0
+yarl==1.18.3
+pydub==0.25.1
+pycparser==2.22
+soundfile==0.13.1
+wcwidth==0.2.13
+groovy==0.1.2
+torch==2.5.1
+anyio==4.9.0
+wandb==0.19.8
+joblib==1.4.2
+fsspec==2024.12.0
+peft==0.15.1
+accelerate==1.5.2
+py-cpuinfo==9.0.0
+uvicorn==0.34.0
+orjson==3.10.16
+Brotli==1.0.9
+rich==13.9.4
+importlib_metadata==8.6.1
+ninja==1.11.1.4
+wheel==0.45.1
+MarkupSafe==3.0.2
+threadpoolctl==3.6.0
+gitdb==4.0.12
+mkl-service==2.4.0
+typing_extensions==4.12.2
+tomli==2.0.1
+zipp==3.19.2
+wheel==0.43.0
+jaraco.text==3.12.1
+packaging==24.2
+autocommand==2.2.2
+jaraco.functools==4.0.1
+jaraco.collections==5.1.0
+platformdirs==4.2.2
+more-itertools==10.3.0
+inflect==7.3.1
+jaraco.context==5.3.0
+typeguard==4.3.0
+backports.tarfile==1.2.0
+importlib_metadata==8.0.0

run-20250329_003923-yubb37lj/files/wandb-metadata.json

@@ -0,0 +1,106 @@
+{
+  "os": "Linux-5.15.0-1040-nvidia-x86_64-with-glibc2.35",
+  "python": "CPython 3.11.11",
+  "startedAt": "2025-03-28T16:39:23.374186Z",
+  "args": [
+    "--local_rank=0",
+    "--model_name_or_path",
+    "/aifs4su/yaodong/hantao/models/llava-v1.6-mistral-7b-hf",
+    "--train_datasets",
+    "/aifs4su/yaodong/hantao/datasets/MMInstruct-GPT4V_mistral-7b_cosi_cut/merged/top1-10",
+    "--train_template",
+    "MM_TI2T_LLAVA",
+    "--train_split",
+    "train",
+    "--train_name",
+    "text-image-to-text",
+    "--output_dir",
+    "../outputs/test_7B",
+    "--save_total_limit",
+    "6",
+    "--train_batch_size",
+    "8",
+    "--epochs",
+    "3"
+  ],
+  "program": "-m align_anything.trainers.text_image_to_text.sft",
+  "git": {
+    "remote": "git@github.com-hantao:PKU-Alignment/align-anything.git",
+    "commit": "106588f9802757a3283c1aff1f33ea9afd737f31"
+  },
+  "email": "2200017789@stu.pku.edu.cn",
+  "root": "../outputs/test_7B",
+  "host": "dgx-092",
+  "executable": "/aifs4su/yaodong/miniconda3/envs/hantao_llama/bin/python",
+  "cpu_count": 112,
+  "cpu_count_logical": 224,
+  "gpu": "NVIDIA H800",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1888556142592",
+      "used": "148607471616"
+    }
+  },
+  "memory": {
+    "total": "2164195454976"
+  },
+  "cpu": {
+    "count": 112,
+    "countLogical": 224
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA H800",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H800",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H800",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H800",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H800",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H800",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H800",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H800",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    }
+  ],
+  "slurm": {
+    "conf": "/cm/shared/apps/slurm/var/etc/slurm/slurm.conf"
+  },
+  "cudaVersion": "12.2"
+}

run-20250329_003923-yubb37lj/logs/debug-core.log

@@ -0,0 +1,7 @@
+{"time":"2025-03-29T00:39:22.471053243+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpjod75rq7/port-18738.txt","pid":18738,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false}
+{"time":"2025-03-29T00:39:22.471952784+08:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":18738}
+{"time":"2025-03-29T00:39:22.471948837+08:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":42047,"Zone":""}}
+{"time":"2025-03-29T00:39:22.652859041+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:57374"}
+{"time":"2025-03-29T00:39:23.375489118+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"yubb37lj","id":"127.0.0.1:57374"}
+{"time":"2025-03-29T00:39:23.590375898+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"yubb37lj","id":"127.0.0.1:57374"}
+{"time":"2025-03-29T00:39:50.477044845+08:00","level":"INFO","msg":"received shutdown signal","signal":15}

run-20250329_003923-yubb37lj/logs/debug-internal.log

@@ -0,0 +1,7 @@
+{"time":"2025-03-29T00:39:23.376972436+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"../outputs/test_7B/wandb/run-20250329_003923-yubb37lj/logs/debug-core.log"}
+{"time":"2025-03-29T00:39:23.590312919+08:00","level":"INFO","msg":"created new stream","id":"yubb37lj"}
+{"time":"2025-03-29T00:39:23.590368384+08:00","level":"INFO","msg":"stream: started","id":"yubb37lj"}
+{"time":"2025-03-29T00:39:23.59039266+08:00","level":"INFO","msg":"handler: started","stream_id":"yubb37lj"}
+{"time":"2025-03-29T00:39:23.59039881+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"yubb37lj"}
+{"time":"2025-03-29T00:39:23.590398926+08:00","level":"INFO","msg":"sender: started","stream_id":"yubb37lj"}
+{"time":"2025-03-29T00:39:23.847802173+08:00","level":"INFO","msg":"Starting system monitor"}

run-20250329_003923-yubb37lj/logs/debug.log

@@ -0,0 +1,22 @@
+2025-03-29 00:39:23,371 INFO    MainThread:18738 [wandb_setup.py:_flush():67] Current SDK version is 0.19.8
+2025-03-29 00:39:23,371 INFO    MainThread:18738 [wandb_setup.py:_flush():67] Configure stats pid to 18738
+2025-03-29 00:39:23,371 INFO    MainThread:18738 [wandb_setup.py:_flush():67] Loading settings from /home/yangyaodong/.config/wandb/settings
+2025-03-29 00:39:23,371 INFO    MainThread:18738 [wandb_setup.py:_flush():67] Loading settings from /aifs4su/yaodong/hantao/align-anything/scripts/wandb/settings
+2025-03-29 00:39:23,371 INFO    MainThread:18738 [wandb_setup.py:_flush():67] Loading settings from environment variables
+2025-03-29 00:39:23,371 INFO    MainThread:18738 [wandb_init.py:setup_run_log_directory():647] Logging user logs to ../outputs/test_7B/wandb/run-20250329_003923-yubb37lj/logs/debug.log
+2025-03-29 00:39:23,371 INFO    MainThread:18738 [wandb_init.py:setup_run_log_directory():648] Logging internal logs to ../outputs/test_7B/wandb/run-20250329_003923-yubb37lj/logs/debug-internal.log
+2025-03-29 00:39:23,371 INFO    MainThread:18738 [wandb_init.py:init():761] calling init triggers
+2025-03-29 00:39:23,371 INFO    MainThread:18738 [wandb_init.py:init():766] wandb.init called with sweep_config: {}
+config: {'train_cfgs': {'save_checkpoint': True, 'load_checkpoint': False, 'ds_cfgs': 'ds_z3_config.json', 'epochs': 3, 'seed': 42, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 1, 'gradient_accumulation_steps': 16, 'gradient_checkpointing': True, 'learning_rate': 2e-05, 'lr_scheduler_type': 'cosine', 'lr_warmup_ratio': 0.03, 'weight_decay': 0.0, 'adam_betas': [0.9, 0.95], 'adam_epsilon': 1e-08, 'bf16': True, 'fp16': False, 'eval_strategy': 'epoch', 'eval_interval': 10, 'freeze_mm_proj': False, 'freeze_vision_tower': True, 'freeze_language_model': False, 'max_grad_norm': 1.0}, 'data_cfgs': {'load_multi_datasets': False, 'train_datasets': '/aifs4su/yaodong/hantao/datasets/MMInstruct-GPT4V_mistral-7b_cosi_cut/merged/top1-10', 'train_template': 'MM_TI2T_LLAVA', 'train_size': {}, 'train_split': 'train', 'train_name': 'text-image-to-text', 'train_data_files': {}, 'train_optional_args': [], 'eval_datasets': {}, 'eval_template': {}, 'eval_name': {}, 'eval_size': {}, 'eval_split': {}, 'eval_subset': {}, 'eval_data_files': {}, 'eval_optional_args': []}, 'logger_cfgs': {'log_type': 'wandb', 'log_project': 'align-anything', 'log_run_name': 'sft', 'output_dir': '../outputs/test_7B', 'cache_dir': {}, 'save_total_limit': 6}, 'model_cfgs': {'model_name_or_path': '/aifs4su/yaodong/hantao/models/llava-v1.6-mistral-7b-hf', 'trust_remote_code': True, 'model_max_length': 2048}, 'special_tokens': {}, '_wandb': {}}
+2025-03-29 00:39:23,371 INFO    MainThread:18738 [wandb_init.py:init():784] starting backend
+2025-03-29 00:39:23,371 INFO    MainThread:18738 [wandb_init.py:init():788] sending inform_init request
+2025-03-29 00:39:23,374 INFO    MainThread:18738 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-03-29 00:39:23,374 INFO    MainThread:18738 [wandb_init.py:init():798] backend started and connected
+2025-03-29 00:39:23,375 INFO    MainThread:18738 [wandb_init.py:init():891] updated telemetry
+2025-03-29 00:39:23,385 INFO    MainThread:18738 [wandb_init.py:init():915] communicating run to backend with 90.0 second timeout
+2025-03-29 00:39:23,845 INFO    MainThread:18738 [wandb_init.py:init():990] starting run threads in backend
+2025-03-29 00:39:23,996 INFO    MainThread:18738 [wandb_run.py:_console_start():2375] atexit reg
+2025-03-29 00:39:23,996 INFO    MainThread:18738 [wandb_run.py:_redirect():2227] redirect: wrap_raw
+2025-03-29 00:39:23,996 INFO    MainThread:18738 [wandb_run.py:_redirect():2292] Wrapping output streams.
+2025-03-29 00:39:23,996 INFO    MainThread:18738 [wandb_run.py:_redirect():2315] Redirects installed.
+2025-03-29 00:39:23,998 INFO    MainThread:18738 [wandb_init.py:init():1032] run started, returning control to user process

run-20250329_004923-vanwhj5e/files/output.log

@@ -0,0 +1,60 @@
+***** Running training *****
+Resuming from checkpoint 3/3 epoch :   0%|                                                                                                                                                                                                                                                                    | 0/7326 [00:00<?, ?it/s]
+Saving model to "../outputs/test_7B/slice_end" ...
+Saving 16-bit model...
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/aifs4su/yaodong/hantao/align-anything/align_anything/trainers/text_image_to_text/sft.py", line 100, in <module>
+    sys.exit(main())
+             ^^^^^^
+  File "/aifs4su/yaodong/hantao/align-anything/align_anything/trainers/text_image_to_text/sft.py", line 96, in main
+    trainer.save()
+  File "/aifs4su/yaodong/hantao/align-anything/align_anything/trainers/text_to_text/sft.py", line 228, in save
+    self.save_transformers(model=model, tag=tag)
+  File "/aifs4su/yaodong/hantao/align-anything/align_anything/trainers/base/supervised_trainer.py", line 435, in save_transformers
+    model.save_16bit_model(output_dir, save_filename=save_file_name)
+  File "/aifs4su/yaodong/miniconda3/envs/hantao_llama/lib/python3.11/site-packages/deepspeed/runtime/engine.py", line 3815, in save_16bit_model
+    state_dict = self._zero3_consolidated_16bit_state_dict(
+                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/aifs4su/yaodong/miniconda3/envs/hantao_llama/lib/python3.11/site-packages/deepspeed/runtime/engine.py", line 3776, in _zero3_consolidated_16bit_state_dict
+    get_layer_state_dict(self.module, prefix="")
+  File "/aifs4su/yaodong/miniconda3/envs/hantao_llama/lib/python3.11/site-packages/deepspeed/runtime/engine.py", line 3769, in get_layer_state_dict
+    get_layer_state_dict(child, prefix + name + ".")
+  File "/aifs4su/yaodong/miniconda3/envs/hantao_llama/lib/python3.11/site-packages/deepspeed/runtime/engine.py", line 3769, in get_layer_state_dict
+    get_layer_state_dict(child, prefix + name + ".")
+  File "/aifs4su/yaodong/miniconda3/envs/hantao_llama/lib/python3.11/site-packages/deepspeed/runtime/engine.py", line 3769, in get_layer_state_dict
+    get_layer_state_dict(child, prefix + name + ".")
+  [Previous line repeated 3 more times]
+  File "/aifs4su/yaodong/miniconda3/envs/hantao_llama/lib/python3.11/site-packages/deepspeed/runtime/engine.py", line 3757, in get_layer_state_dict
+    state_dict[key] = param.detach().cpu()
+                      ^^^^^^^^^^^^^^^^^^^^
+KeyboardInterrupt
+[rank0]: Traceback (most recent call last):
+[rank0]:   File "<frozen runpy>", line 198, in _run_module_as_main
+[rank0]:   File "<frozen runpy>", line 88, in _run_code
+[rank0]:   File "/aifs4su/yaodong/hantao/align-anything/align_anything/trainers/text_image_to_text/sft.py", line 100, in <module>
+[rank0]:     sys.exit(main())
+[rank0]:              ^^^^^^
+[rank0]:   File "/aifs4su/yaodong/hantao/align-anything/align_anything/trainers/text_image_to_text/sft.py", line 96, in main
+[rank0]:     trainer.save()
+[rank0]:   File "/aifs4su/yaodong/hantao/align-anything/align_anything/trainers/text_to_text/sft.py", line 228, in save
+[rank0]:     self.save_transformers(model=model, tag=tag)
+[rank0]:   File "/aifs4su/yaodong/hantao/align-anything/align_anything/trainers/base/supervised_trainer.py", line 435, in save_transformers
+[rank0]:     model.save_16bit_model(output_dir, save_filename=save_file_name)
+[rank0]:   File "/aifs4su/yaodong/miniconda3/envs/hantao_llama/lib/python3.11/site-packages/deepspeed/runtime/engine.py", line 3815, in save_16bit_model
+[rank0]:     state_dict = self._zero3_consolidated_16bit_state_dict(
+[rank0]:                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:   File "/aifs4su/yaodong/miniconda3/envs/hantao_llama/lib/python3.11/site-packages/deepspeed/runtime/engine.py", line 3776, in _zero3_consolidated_16bit_state_dict
+[rank0]:     get_layer_state_dict(self.module, prefix="")
+[rank0]:   File "/aifs4su/yaodong/miniconda3/envs/hantao_llama/lib/python3.11/site-packages/deepspeed/runtime/engine.py", line 3769, in get_layer_state_dict
+[rank0]:     get_layer_state_dict(child, prefix + name + ".")
+[rank0]:   File "/aifs4su/yaodong/miniconda3/envs/hantao_llama/lib/python3.11/site-packages/deepspeed/runtime/engine.py", line 3769, in get_layer_state_dict
+[rank0]:     get_layer_state_dict(child, prefix + name + ".")
+[rank0]:   File "/aifs4su/yaodong/miniconda3/envs/hantao_llama/lib/python3.11/site-packages/deepspeed/runtime/engine.py", line 3769, in get_layer_state_dict
+[rank0]:     get_layer_state_dict(child, prefix + name + ".")
+[rank0]:   [Previous line repeated 3 more times]
+[rank0]:   File "/aifs4su/yaodong/miniconda3/envs/hantao_llama/lib/python3.11/site-packages/deepspeed/runtime/engine.py", line 3757, in get_layer_state_dict
+[rank0]:     state_dict[key] = param.detach().cpu()
+[rank0]:                       ^^^^^^^^^^^^^^^^^^^^
+[rank0]: KeyboardInterrupt

run-20250329_004923-vanwhj5e/files/requirements.txt

@@ -0,0 +1,167 @@
+maskrcnn_benchmark==0.0.0
+webdataset==0.2.111
+websockets==15.0.1
+typer==0.15.2
+blobfile==3.0.0
+pooch==1.8.2
+python-dateutil==2.9.0.post0
+gmpy2==2.2.1
+httpcore==1.0.7
+charset-normalizer==3.3.2
+torchlibrosa==0.1.0
+multiprocess==0.70.16
+Werkzeug==3.1.3
+aiofiles==23.2.1
+six==1.17.0
+typing_extensions==4.12.2
+psutil==7.0.0
+frozenlist==1.5.0
+einops==0.8.1
+flash_attn==2.7.4.post1
+PySocks==1.7.1
+regex==2024.11.6
+markdown-it-py==3.0.0
+ruff==0.11.2
+docker-pycreds==0.4.0
+protobuf==5.29.4
+resampy==0.4.3
+aiohappyeyeballs==2.6.1
+httpx==0.28.1
+encodec==0.1.1
+ffmpy==0.5.0
+mkl_random==1.2.8
+soxr==0.5.0.post1
+absl-py==2.2.1
+networkx==3.4.2
+h5py==3.13.0
+hjson==3.1.0
+tensorboard==2.19.0
+aiosignal==1.3.2
+pip==25.0
+triton==3.1.0
+zipp==3.21.0
+ftfy==6.3.1
+attrs==25.3.0
+requests==2.32.3
+progressbar==2.5
+sniffio==1.3.1
+lxml==5.3.1
+starlette==0.46.1
+Markdown==3.7
+mdurl==0.1.2
+torchaudio==2.5.1
+safetensors==0.5.3
+opencv-python==4.6.0.66
+torchvision==0.20.1
+shellingham==1.5.4
+gradio==5.23.1
+timm==1.0.15
+multidict==6.2.0
+semantic-version==2.10.0
+numba==0.60.0
+gradio_client==1.8.0
+pydantic_core==2.33.0
+dill==0.3.8
+msgpack==1.1.0
+sentry-sdk==2.24.1
+grpcio==1.71.0
+cffi==1.17.1
+PyYAML==6.0.2
+tensorboard-data-server==0.7.2
+fastapi==0.115.12
+lazy_loader==0.4
+mkl_fft==1.3.11
+annotated-types==0.7.0
+scikit-learn==1.6.1
+wget==3.2
+setuptools==75.8.0
+certifi==2025.1.31
+click==8.1.8
+laion_clap==1.1.5
+Pygments==2.19.1
+tomlkit==0.13.2
+idna==3.7
+propcache==0.3.1
+platformdirs==4.3.7
+align-anything==0.0.1.dev0
+deepspeed==0.16.5
+smmap==5.0.2
+pillow==11.1.0
+typing-inspection==0.4.0
+braceexpand==0.1.7
+decorator==5.2.1
+pandas==2.2.3
+huggingface-hub==0.29.3
+pyarrow==19.0.1
+tokenizers==0.21.1
+GitPython==3.1.44
+xxhash==3.5.0
+packaging==24.2
+numpy==1.23.4
+setproctitle==1.3.5
+llvmlite==0.43.0
+tiktoken==0.9.0
+mpmath==1.3.0
+nvidia-ml-py==12.570.86
+pydantic==2.11.0
+datasets==3.5.0
+librosa==0.11.0
+frechet_audio_distance==0.3.1
+sympy==1.13.1
+safehttpx==0.1.6
+Jinja2==3.1.6
+h11==0.14.0
+aiohttp==3.11.14
+diffusers==0.32.2
+tqdm==4.67.1
+filelock==3.13.1
+transformers==4.51.0.dev0
+scipy==1.10.1
+audioread==3.0.1
+sentencepiece==0.2.0
+pytz==2025.2
+tzdata==2025.2
+python-multipart==0.0.20
+urllib3==2.3.0
+pycryptodomex==3.22.0
+yarl==1.18.3
+pydub==0.25.1
+pycparser==2.22
+soundfile==0.13.1
+wcwidth==0.2.13
+groovy==0.1.2
+torch==2.5.1
+anyio==4.9.0
+wandb==0.19.8
+joblib==1.4.2
+fsspec==2024.12.0
+peft==0.15.1
+accelerate==1.5.2
+py-cpuinfo==9.0.0
+uvicorn==0.34.0
+orjson==3.10.16
+Brotli==1.0.9
+rich==13.9.4
+importlib_metadata==8.6.1
+ninja==1.11.1.4
+wheel==0.45.1
+MarkupSafe==3.0.2
+threadpoolctl==3.6.0
+gitdb==4.0.12
+mkl-service==2.4.0
+typing_extensions==4.12.2
+tomli==2.0.1
+zipp==3.19.2
+wheel==0.43.0
+jaraco.text==3.12.1
+packaging==24.2
+autocommand==2.2.2
+jaraco.functools==4.0.1
+jaraco.collections==5.1.0
+platformdirs==4.2.2
+more-itertools==10.3.0
+inflect==7.3.1
+jaraco.context==5.3.0
+typeguard==4.3.0
+backports.tarfile==1.2.0
+importlib_metadata==8.0.0

run-20250329_004923-vanwhj5e/files/wandb-metadata.json

@@ -0,0 +1,106 @@
+{
+  "os": "Linux-5.15.0-1040-nvidia-x86_64-with-glibc2.35",
+  "python": "CPython 3.11.11",
+  "startedAt": "2025-03-28T16:49:23.693460Z",
+  "args": [
+    "--local_rank=0",
+    "--model_name_or_path",
+    "/aifs4su/yaodong/hantao/models/llava-v1.6-mistral-7b-hf",
+    "--train_datasets",
+    "/aifs4su/yaodong/hantao/datasets/MMInstruct-GPT4V_mistral-7b_cosi_cut/merged/top1-10",
+    "--train_template",
+    "MM_TI2T_LLAVA",
+    "--train_split",
+    "train",
+    "--train_name",
+    "text-image-to-text",
+    "--output_dir",
+    "../outputs/test_7B",
+    "--save_total_limit",
+    "6",
+    "--train_batch_size",
+    "8",
+    "--epochs",
+    "3"
+  ],
+  "program": "-m align_anything.trainers.text_image_to_text.sft",
+  "git": {
+    "remote": "git@github.com-hantao:PKU-Alignment/align-anything.git",
+    "commit": "106588f9802757a3283c1aff1f33ea9afd737f31"
+  },
+  "email": "2200017789@stu.pku.edu.cn",
+  "root": "../outputs/test_7B",
+  "host": "dgx-092",
+  "executable": "/aifs4su/yaodong/miniconda3/envs/hantao_llama/bin/python",
+  "cpu_count": 112,
+  "cpu_count_logical": 224,
+  "gpu": "NVIDIA H800",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1888556142592",
+      "used": "148608499712"
+    }
+  },
+  "memory": {
+    "total": "2164195454976"
+  },
+  "cpu": {
+    "count": 112,
+    "countLogical": 224
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA H800",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H800",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H800",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H800",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H800",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H800",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H800",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H800",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    }
+  ],
+  "slurm": {
+    "conf": "/cm/shared/apps/slurm/var/etc/slurm/slurm.conf"
+  },
+  "cudaVersion": "12.2"
+}

run-20250329_004923-vanwhj5e/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":4}}

run-20250329_004923-vanwhj5e/logs/debug-core.log

@@ -0,0 +1,7 @@
+{"time":"2025-03-29T00:49:23.0903988+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp1bwuylg2/port-35456.txt","pid":35456,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false}
+{"time":"2025-03-29T00:49:23.091269653+08:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":35456}
+{"time":"2025-03-29T00:49:23.091244341+08:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":40795,"Zone":""}}
+{"time":"2025-03-29T00:49:23.270609752+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:54946"}
+{"time":"2025-03-29T00:49:23.694940245+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"vanwhj5e","id":"127.0.0.1:54946"}
+{"time":"2025-03-29T00:49:23.912684251+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"vanwhj5e","id":"127.0.0.1:54946"}
+{"time":"2025-03-29T00:49:27.869501123+08:00","level":"INFO","msg":"received shutdown signal","signal":15}

run-20250329_004923-vanwhj5e/logs/debug-internal.log

@@ -0,0 +1,9 @@
+{"time":"2025-03-29T00:49:23.696239084+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"../outputs/test_7B/wandb/run-20250329_004923-vanwhj5e/logs/debug-core.log"}
+{"time":"2025-03-29T00:49:23.912592778+08:00","level":"INFO","msg":"created new stream","id":"vanwhj5e"}
+{"time":"2025-03-29T00:49:23.91267373+08:00","level":"INFO","msg":"stream: started","id":"vanwhj5e"}
+{"time":"2025-03-29T00:49:23.912692782+08:00","level":"INFO","msg":"handler: started","stream_id":"vanwhj5e"}
+{"time":"2025-03-29T00:49:23.912700862+08:00","level":"INFO","msg":"sender: started","stream_id":"vanwhj5e"}
+{"time":"2025-03-29T00:49:23.912703724+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"vanwhj5e"}
+{"time":"2025-03-29T00:49:24.229784705+08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2025-03-29T00:49:27.854433357+08:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2025-03-29T00:49:27.855141086+08:00","level":"INFO","msg":"Stopped system monitor"}

run-20250329_004923-vanwhj5e/logs/debug.log

@@ -0,0 +1,26 @@
+2025-03-29 00:49:23,690 INFO    MainThread:35456 [wandb_setup.py:_flush():67] Current SDK version is 0.19.8
+2025-03-29 00:49:23,690 INFO    MainThread:35456 [wandb_setup.py:_flush():67] Configure stats pid to 35456
+2025-03-29 00:49:23,690 INFO    MainThread:35456 [wandb_setup.py:_flush():67] Loading settings from /home/yangyaodong/.config/wandb/settings
+2025-03-29 00:49:23,690 INFO    MainThread:35456 [wandb_setup.py:_flush():67] Loading settings from /aifs4su/yaodong/hantao/align-anything/scripts/wandb/settings
+2025-03-29 00:49:23,690 INFO    MainThread:35456 [wandb_setup.py:_flush():67] Loading settings from environment variables
+2025-03-29 00:49:23,690 INFO    MainThread:35456 [wandb_init.py:setup_run_log_directory():647] Logging user logs to ../outputs/test_7B/wandb/run-20250329_004923-vanwhj5e/logs/debug.log
+2025-03-29 00:49:23,690 INFO    MainThread:35456 [wandb_init.py:setup_run_log_directory():648] Logging internal logs to ../outputs/test_7B/wandb/run-20250329_004923-vanwhj5e/logs/debug-internal.log
+2025-03-29 00:49:23,690 INFO    MainThread:35456 [wandb_init.py:init():761] calling init triggers
+2025-03-29 00:49:23,690 INFO    MainThread:35456 [wandb_init.py:init():766] wandb.init called with sweep_config: {}
+config: {'train_cfgs': {'save_checkpoint': False, 'load_checkpoint': False, 'ds_cfgs': 'ds_z3_config.json', 'epochs': 3, 'seed': 42, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 1, 'gradient_accumulation_steps': 16, 'gradient_checkpointing': True, 'learning_rate': 2e-05, 'lr_scheduler_type': 'cosine', 'lr_warmup_ratio': 0.03, 'weight_decay': 0.0, 'adam_betas': [0.9, 0.95], 'adam_epsilon': 1e-08, 'bf16': True, 'fp16': False, 'eval_strategy': 'epoch', 'eval_interval': 10, 'freeze_mm_proj': False, 'freeze_vision_tower': True, 'freeze_language_model': False, 'max_grad_norm': 1.0}, 'data_cfgs': {'load_multi_datasets': False, 'train_datasets': '/aifs4su/yaodong/hantao/datasets/MMInstruct-GPT4V_mistral-7b_cosi_cut/merged/top1-10', 'train_template': 'MM_TI2T_LLAVA', 'train_size': {}, 'train_split': 'train', 'train_name': 'text-image-to-text', 'train_data_files': {}, 'train_optional_args': [], 'eval_datasets': {}, 'eval_template': {}, 'eval_name': {}, 'eval_size': {}, 'eval_split': {}, 'eval_subset': {}, 'eval_data_files': {}, 'eval_optional_args': []}, 'logger_cfgs': {'log_type': 'wandb', 'log_project': 'align-anything', 'log_run_name': 'sft', 'output_dir': '../outputs/test_7B', 'cache_dir': {}, 'save_total_limit': 6}, 'model_cfgs': {'model_name_or_path': '/aifs4su/yaodong/hantao/models/llava-v1.6-mistral-7b-hf', 'trust_remote_code': True, 'model_max_length': 2048}, 'special_tokens': {}, '_wandb': {}}
+2025-03-29 00:49:23,690 INFO    MainThread:35456 [wandb_init.py:init():784] starting backend
+2025-03-29 00:49:23,690 INFO    MainThread:35456 [wandb_init.py:init():788] sending inform_init request
+2025-03-29 00:49:23,693 INFO    MainThread:35456 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-03-29 00:49:23,693 INFO    MainThread:35456 [wandb_init.py:init():798] backend started and connected
+2025-03-29 00:49:23,694 INFO    MainThread:35456 [wandb_init.py:init():891] updated telemetry
+2025-03-29 00:49:23,705 INFO    MainThread:35456 [wandb_init.py:init():915] communicating run to backend with 90.0 second timeout
+2025-03-29 00:49:24,227 INFO    MainThread:35456 [wandb_init.py:init():990] starting run threads in backend
+2025-03-29 00:49:24,382 INFO    MainThread:35456 [wandb_run.py:_console_start():2375] atexit reg
+2025-03-29 00:49:24,382 INFO    MainThread:35456 [wandb_run.py:_redirect():2227] redirect: wrap_raw
+2025-03-29 00:49:24,382 INFO    MainThread:35456 [wandb_run.py:_redirect():2292] Wrapping output streams.
+2025-03-29 00:49:24,382 INFO    MainThread:35456 [wandb_run.py:_redirect():2315] Redirects installed.
+2025-03-29 00:49:24,384 INFO    MainThread:35456 [wandb_init.py:init():1032] run started, returning control to user process
+2025-03-29 00:49:27,852 INFO    MainThread:35456 [wandb_run.py:_finish():2112] finishing run htlou/align-anything/vanwhj5e
+2025-03-29 00:49:27,853 INFO    MainThread:35456 [wandb_run.py:_atexit_cleanup():2340] got exitcode: 0
+2025-03-29 00:49:27,853 INFO    MainThread:35456 [wandb_run.py:_restore():2322] restore
+2025-03-29 00:49:27,854 INFO    MainThread:35456 [wandb_run.py:_restore():2328] restore done

run-20250329_005139-6x2eqgtz/files/requirements.txt

@@ -0,0 +1,167 @@
+maskrcnn_benchmark==0.0.0
+webdataset==0.2.111
+websockets==15.0.1
+typer==0.15.2
+blobfile==3.0.0
+pooch==1.8.2
+python-dateutil==2.9.0.post0
+gmpy2==2.2.1
+httpcore==1.0.7
+charset-normalizer==3.3.2
+torchlibrosa==0.1.0
+multiprocess==0.70.16
+Werkzeug==3.1.3
+aiofiles==23.2.1
+six==1.17.0
+typing_extensions==4.12.2
+psutil==7.0.0
+frozenlist==1.5.0
+einops==0.8.1
+flash_attn==2.7.4.post1
+PySocks==1.7.1
+regex==2024.11.6
+markdown-it-py==3.0.0
+ruff==0.11.2
+docker-pycreds==0.4.0
+protobuf==5.29.4
+resampy==0.4.3
+aiohappyeyeballs==2.6.1
+httpx==0.28.1
+encodec==0.1.1
+ffmpy==0.5.0
+mkl_random==1.2.8
+soxr==0.5.0.post1
+absl-py==2.2.1
+networkx==3.4.2
+h5py==3.13.0
+hjson==3.1.0
+tensorboard==2.19.0
+aiosignal==1.3.2
+pip==25.0
+triton==3.1.0
+zipp==3.21.0
+ftfy==6.3.1
+attrs==25.3.0
+requests==2.32.3
+progressbar==2.5
+sniffio==1.3.1
+lxml==5.3.1
+starlette==0.46.1
+Markdown==3.7
+mdurl==0.1.2
+torchaudio==2.5.1
+safetensors==0.5.3
+opencv-python==4.6.0.66
+torchvision==0.20.1
+shellingham==1.5.4
+gradio==5.23.1
+timm==1.0.15
+multidict==6.2.0
+semantic-version==2.10.0
+numba==0.60.0
+gradio_client==1.8.0
+pydantic_core==2.33.0
+dill==0.3.8
+msgpack==1.1.0
+sentry-sdk==2.24.1
+grpcio==1.71.0
+cffi==1.17.1
+PyYAML==6.0.2
+tensorboard-data-server==0.7.2
+fastapi==0.115.12
+lazy_loader==0.4
+mkl_fft==1.3.11
+annotated-types==0.7.0
+scikit-learn==1.6.1
+wget==3.2
+setuptools==75.8.0
+certifi==2025.1.31
+click==8.1.8
+laion_clap==1.1.5
+Pygments==2.19.1
+tomlkit==0.13.2
+idna==3.7
+propcache==0.3.1
+platformdirs==4.3.7
+align-anything==0.0.1.dev0
+deepspeed==0.16.5
+smmap==5.0.2
+pillow==11.1.0
+typing-inspection==0.4.0
+braceexpand==0.1.7
+decorator==5.2.1
+pandas==2.2.3
+huggingface-hub==0.29.3
+pyarrow==19.0.1
+tokenizers==0.21.1
+GitPython==3.1.44
+xxhash==3.5.0
+packaging==24.2
+numpy==1.23.4
+setproctitle==1.3.5
+llvmlite==0.43.0
+tiktoken==0.9.0
+mpmath==1.3.0
+nvidia-ml-py==12.570.86
+pydantic==2.11.0
+datasets==3.5.0
+librosa==0.11.0
+frechet_audio_distance==0.3.1
+sympy==1.13.1
+safehttpx==0.1.6
+Jinja2==3.1.6
+h11==0.14.0
+aiohttp==3.11.14
+diffusers==0.32.2
+tqdm==4.67.1
+filelock==3.13.1
+transformers==4.51.0.dev0
+scipy==1.10.1
+audioread==3.0.1
+sentencepiece==0.2.0
+pytz==2025.2
+tzdata==2025.2
+python-multipart==0.0.20
+urllib3==2.3.0
+pycryptodomex==3.22.0
+yarl==1.18.3
+pydub==0.25.1
+pycparser==2.22
+soundfile==0.13.1
+wcwidth==0.2.13
+groovy==0.1.2
+torch==2.5.1
+anyio==4.9.0
+wandb==0.19.8
+joblib==1.4.2
+fsspec==2024.12.0
+peft==0.15.1
+accelerate==1.5.2
+py-cpuinfo==9.0.0
+uvicorn==0.34.0
+orjson==3.10.16
+Brotli==1.0.9
+rich==13.9.4
+importlib_metadata==8.6.1
+ninja==1.11.1.4
+wheel==0.45.1
+MarkupSafe==3.0.2
+threadpoolctl==3.6.0
+gitdb==4.0.12
+mkl-service==2.4.0
+typing_extensions==4.12.2
+tomli==2.0.1
+zipp==3.19.2
+wheel==0.43.0
+jaraco.text==3.12.1
+packaging==24.2
+autocommand==2.2.2
+jaraco.functools==4.0.1
+jaraco.collections==5.1.0
+platformdirs==4.2.2
+more-itertools==10.3.0
+inflect==7.3.1
+jaraco.context==5.3.0
+typeguard==4.3.0
+backports.tarfile==1.2.0
+importlib_metadata==8.0.0

run-20250329_005139-6x2eqgtz/files/wandb-metadata.json

@@ -0,0 +1,35 @@
+{
+  "os": "Linux-5.15.0-1040-nvidia-x86_64-with-glibc2.35",
+  "python": "CPython 3.11.11",
+  "startedAt": "2025-03-28T16:51:39.067886Z",
+  "args": [
+    "--local_rank=0",
+    "--model_name_or_path",
+    "/aifs4su/yaodong/hantao/models/llava-v1.6-mistral-7b-hf",
+    "--train_datasets",
+    "/aifs4su/yaodong/hantao/datasets/MMInstruct-GPT4V_mistral-7b_cosi_cut/merged/top1-10",
+    "--train_template",
+    "MM_TI2T_LLAVA",
+    "--train_split",
+    "train",
+    "--train_name",
+    "text-image-to-text",
+    "--output_dir",
+    "../outputs/test_7B",
+    "--save_total_limit",
+    "6",
+    "--train_batch_size",
+    "8",
+    "--epochs",
+    "3"
+  ],
+  "program": "-m align_anything.trainers.text_image_to_text.sft",
+  "git": {
+    "remote": "git@github.com-hantao:PKU-Alignment/align-anything.git",
+    "commit": "106588f9802757a3283c1aff1f33ea9afd737f31"
+  },
+  "email": "2200017789@stu.pku.edu.cn",
+  "root": "../outputs/test_7B",
+  "host": "dgx-092",
+  "executable": "/aifs4su/yaodong/miniconda3/envs/hantao_llama/bin/python"
+}

run-20250329_005139-6x2eqgtz/logs/debug-core.log

@@ -0,0 +1,7 @@
+{"time":"2025-03-29T00:51:38.479231525+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpb6xx_4px/port-42596.txt","pid":42596,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false}
+{"time":"2025-03-29T00:51:38.480170741+08:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":42596}
+{"time":"2025-03-29T00:51:38.480164974+08:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":35441,"Zone":""}}
+{"time":"2025-03-29T00:51:38.663800746+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:50004"}
+{"time":"2025-03-29T00:51:39.069534671+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"6x2eqgtz","id":"127.0.0.1:50004"}
+{"time":"2025-03-29T00:51:39.285743333+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"6x2eqgtz","id":"127.0.0.1:50004"}
+{"time":"2025-03-29T00:51:40.320100827+08:00","level":"INFO","msg":"received shutdown signal","signal":15}

run-20250329_005139-6x2eqgtz/logs/debug-internal.log

@@ -0,0 +1,8 @@
+{"time":"2025-03-29T00:51:39.071049577+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"../outputs/test_7B/wandb/run-20250329_005139-6x2eqgtz/logs/debug-core.log"}
+{"time":"2025-03-29T00:51:39.285637637+08:00","level":"INFO","msg":"created new stream","id":"6x2eqgtz"}
+{"time":"2025-03-29T00:51:39.285734961+08:00","level":"INFO","msg":"stream: started","id":"6x2eqgtz"}
+{"time":"2025-03-29T00:51:39.285766391+08:00","level":"INFO","msg":"handler: started","stream_id":"6x2eqgtz"}
+{"time":"2025-03-29T00:51:39.286029925+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"6x2eqgtz"}
+{"time":"2025-03-29T00:51:39.285781884+08:00","level":"INFO","msg":"sender: started","stream_id":"6x2eqgtz"}
+{"time":"2025-03-29T00:51:39.613194812+08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2025-03-29T00:51:39.846153883+08:00","level":"INFO","msg":"Stopping system monitor"}

run-20250329_005139-6x2eqgtz/logs/debug.log

@@ -0,0 +1,26 @@
+2025-03-29 00:51:39,064 INFO    MainThread:42596 [wandb_setup.py:_flush():67] Current SDK version is 0.19.8
+2025-03-29 00:51:39,064 INFO    MainThread:42596 [wandb_setup.py:_flush():67] Configure stats pid to 42596
+2025-03-29 00:51:39,064 INFO    MainThread:42596 [wandb_setup.py:_flush():67] Loading settings from /home/yangyaodong/.config/wandb/settings
+2025-03-29 00:51:39,064 INFO    MainThread:42596 [wandb_setup.py:_flush():67] Loading settings from /aifs4su/yaodong/hantao/align-anything/scripts/wandb/settings
+2025-03-29 00:51:39,064 INFO    MainThread:42596 [wandb_setup.py:_flush():67] Loading settings from environment variables
+2025-03-29 00:51:39,065 INFO    MainThread:42596 [wandb_init.py:setup_run_log_directory():647] Logging user logs to ../outputs/test_7B/wandb/run-20250329_005139-6x2eqgtz/logs/debug.log
+2025-03-29 00:51:39,065 INFO    MainThread:42596 [wandb_init.py:setup_run_log_directory():648] Logging internal logs to ../outputs/test_7B/wandb/run-20250329_005139-6x2eqgtz/logs/debug-internal.log
+2025-03-29 00:51:39,065 INFO    MainThread:42596 [wandb_init.py:init():761] calling init triggers
+2025-03-29 00:51:39,065 INFO    MainThread:42596 [wandb_init.py:init():766] wandb.init called with sweep_config: {}
+config: {'train_cfgs': {'save_checkpoint': False, 'load_checkpoint': False, 'ds_cfgs': 'ds_z3_config.json', 'epochs': 3, 'seed': 42, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 1, 'gradient_accumulation_steps': 16, 'gradient_checkpointing': True, 'learning_rate': 2e-05, 'lr_scheduler_type': 'cosine', 'lr_warmup_ratio': 0.03, 'weight_decay': 0.0, 'adam_betas': [0.9, 0.95], 'adam_epsilon': 1e-08, 'bf16': True, 'fp16': False, 'eval_strategy': 'epoch', 'eval_interval': 10, 'freeze_mm_proj': False, 'freeze_vision_tower': True, 'freeze_language_model': False, 'max_grad_norm': 1.0}, 'data_cfgs': {'load_multi_datasets': False, 'train_datasets': '/aifs4su/yaodong/hantao/datasets/MMInstruct-GPT4V_mistral-7b_cosi_cut/merged/top1-10', 'train_template': 'MM_TI2T_LLAVA', 'train_size': {}, 'train_split': 'train', 'train_name': 'text-image-to-text', 'train_data_files': {}, 'train_optional_args': [], 'eval_datasets': {}, 'eval_template': {}, 'eval_name': {}, 'eval_size': {}, 'eval_split': {}, 'eval_subset': {}, 'eval_data_files': {}, 'eval_optional_args': []}, 'logger_cfgs': {'log_type': 'wandb', 'log_project': 'align-anything', 'log_run_name': 'sft', 'output_dir': '../outputs/test_7B', 'cache_dir': {}, 'save_total_limit': 6}, 'model_cfgs': {'model_name_or_path': '/aifs4su/yaodong/hantao/models/llava-v1.6-mistral-7b-hf', 'trust_remote_code': True, 'model_max_length': 2048}, 'special_tokens': {}, '_wandb': {}}
+2025-03-29 00:51:39,065 INFO    MainThread:42596 [wandb_init.py:init():784] starting backend
+2025-03-29 00:51:39,065 INFO    MainThread:42596 [wandb_init.py:init():788] sending inform_init request
+2025-03-29 00:51:39,067 INFO    MainThread:42596 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-03-29 00:51:39,067 INFO    MainThread:42596 [wandb_init.py:init():798] backend started and connected
+2025-03-29 00:51:39,068 INFO    MainThread:42596 [wandb_init.py:init():891] updated telemetry
+2025-03-29 00:51:39,080 INFO    MainThread:42596 [wandb_init.py:init():915] communicating run to backend with 90.0 second timeout
+2025-03-29 00:51:39,610 INFO    MainThread:42596 [wandb_init.py:init():990] starting run threads in backend
+2025-03-29 00:51:39,802 INFO    MainThread:42596 [wandb_run.py:_console_start():2375] atexit reg
+2025-03-29 00:51:39,803 INFO    MainThread:42596 [wandb_run.py:_redirect():2227] redirect: wrap_raw
+2025-03-29 00:51:39,803 INFO    MainThread:42596 [wandb_run.py:_redirect():2292] Wrapping output streams.
+2025-03-29 00:51:39,803 INFO    MainThread:42596 [wandb_run.py:_redirect():2315] Redirects installed.
+2025-03-29 00:51:39,805 INFO    MainThread:42596 [wandb_init.py:init():1032] run started, returning control to user process
+2025-03-29 00:51:39,816 INFO    MainThread:42596 [wandb_run.py:_finish():2112] finishing run htlou/align-anything/6x2eqgtz
+2025-03-29 00:51:39,816 INFO    MainThread:42596 [wandb_run.py:_atexit_cleanup():2340] got exitcode: 0
+2025-03-29 00:51:39,816 INFO    MainThread:42596 [wandb_run.py:_restore():2322] restore
+2025-03-29 00:51:39,816 INFO    MainThread:42596 [wandb_run.py:_restore():2328] restore done

run-20250329_005425-3al6iztu/files/output.log

@@ -0,0 +1,13 @@
+***** Running training *****
+Resuming from checkpoint 1/3 epoch :   0%|                                                                                                                                                                                                                                                                    | 0/7326 [00:00<?, ?it/s]Traceback (most recent call last):
+Train dataloader:  <torch.utils.data.dataloader.DataLoader object at 0x1551083766d0>
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/aifs4su/yaodong/hantao/align-anything/align_anything/trainers/text_image_to_text/sft.py", line 100, in <module>
+    sys.exit(main())
+             ^^^^^^
+  File "/aifs4su/yaodong/hantao/align-anything/align_anything/trainers/text_image_to_text/sft.py", line 95, in main
+    trainer.train()
+  File "/aifs4su/yaodong/hantao/align-anything/align_anything/trainers/text_to_text/sft.py", line 140, in train
+    print('First batch: ', enumerate(self.train_dataloader)[0])
+                           ~~~~~~~~~~~~~~~~~~~~

run-20250329_005425-3al6iztu/files/requirements.txt

@@ -0,0 +1,167 @@
+maskrcnn_benchmark==0.0.0
+webdataset==0.2.111
+websockets==15.0.1
+typer==0.15.2
+blobfile==3.0.0
+pooch==1.8.2
+python-dateutil==2.9.0.post0
+gmpy2==2.2.1
+httpcore==1.0.7
+charset-normalizer==3.3.2
+torchlibrosa==0.1.0
+multiprocess==0.70.16
+Werkzeug==3.1.3
+aiofiles==23.2.1
+six==1.17.0
+typing_extensions==4.12.2
+psutil==7.0.0
+frozenlist==1.5.0
+einops==0.8.1
+flash_attn==2.7.4.post1
+PySocks==1.7.1
+regex==2024.11.6
+markdown-it-py==3.0.0
+ruff==0.11.2
+docker-pycreds==0.4.0
+protobuf==5.29.4
+resampy==0.4.3
+aiohappyeyeballs==2.6.1
+httpx==0.28.1
+encodec==0.1.1
+ffmpy==0.5.0
+mkl_random==1.2.8
+soxr==0.5.0.post1
+absl-py==2.2.1
+networkx==3.4.2
+h5py==3.13.0
+hjson==3.1.0
+tensorboard==2.19.0
+aiosignal==1.3.2
+pip==25.0
+triton==3.1.0
+zipp==3.21.0
+ftfy==6.3.1
+attrs==25.3.0
+requests==2.32.3
+progressbar==2.5
+sniffio==1.3.1
+lxml==5.3.1
+starlette==0.46.1
+Markdown==3.7
+mdurl==0.1.2
+torchaudio==2.5.1
+safetensors==0.5.3
+opencv-python==4.6.0.66
+torchvision==0.20.1
+shellingham==1.5.4
+gradio==5.23.1
+timm==1.0.15
+multidict==6.2.0
+semantic-version==2.10.0
+numba==0.60.0
+gradio_client==1.8.0
+pydantic_core==2.33.0
+dill==0.3.8
+msgpack==1.1.0
+sentry-sdk==2.24.1
+grpcio==1.71.0
+cffi==1.17.1
+PyYAML==6.0.2
+tensorboard-data-server==0.7.2
+fastapi==0.115.12
+lazy_loader==0.4
+mkl_fft==1.3.11
+annotated-types==0.7.0
+scikit-learn==1.6.1
+wget==3.2
+setuptools==75.8.0
+certifi==2025.1.31
+click==8.1.8
+laion_clap==1.1.5
+Pygments==2.19.1
+tomlkit==0.13.2
+idna==3.7
+propcache==0.3.1
+platformdirs==4.3.7
+align-anything==0.0.1.dev0
+deepspeed==0.16.5
+smmap==5.0.2
+pillow==11.1.0
+typing-inspection==0.4.0
+braceexpand==0.1.7
+decorator==5.2.1
+pandas==2.2.3
+huggingface-hub==0.29.3
+pyarrow==19.0.1
+tokenizers==0.21.1
+GitPython==3.1.44
+xxhash==3.5.0
+packaging==24.2
+numpy==1.23.4
+setproctitle==1.3.5
+llvmlite==0.43.0
+tiktoken==0.9.0
+mpmath==1.3.0
+nvidia-ml-py==12.570.86
+pydantic==2.11.0
+datasets==3.5.0
+librosa==0.11.0
+frechet_audio_distance==0.3.1
+sympy==1.13.1
+safehttpx==0.1.6
+Jinja2==3.1.6
+h11==0.14.0
+aiohttp==3.11.14
+diffusers==0.32.2
+tqdm==4.67.1
+filelock==3.13.1
+transformers==4.51.0.dev0
+scipy==1.10.1
+audioread==3.0.1
+sentencepiece==0.2.0
+pytz==2025.2
+tzdata==2025.2
+python-multipart==0.0.20
+urllib3==2.3.0
+pycryptodomex==3.22.0
+yarl==1.18.3
+pydub==0.25.1
+pycparser==2.22
+soundfile==0.13.1
+wcwidth==0.2.13
+groovy==0.1.2
+torch==2.5.1
+anyio==4.9.0
+wandb==0.19.8
+joblib==1.4.2
+fsspec==2024.12.0
+peft==0.15.1
+accelerate==1.5.2
+py-cpuinfo==9.0.0
+uvicorn==0.34.0
+orjson==3.10.16
+Brotli==1.0.9
+rich==13.9.4
+importlib_metadata==8.6.1
+ninja==1.11.1.4
+wheel==0.45.1
+MarkupSafe==3.0.2
+threadpoolctl==3.6.0
+gitdb==4.0.12
+mkl-service==2.4.0
+typing_extensions==4.12.2
+tomli==2.0.1
+zipp==3.19.2
+wheel==0.43.0
+jaraco.text==3.12.1
+packaging==24.2
+autocommand==2.2.2
+jaraco.functools==4.0.1
+jaraco.collections==5.1.0
+platformdirs==4.2.2
+more-itertools==10.3.0
+inflect==7.3.1
+jaraco.context==5.3.0
+typeguard==4.3.0
+backports.tarfile==1.2.0
+importlib_metadata==8.0.0

run-20250329_005425-3al6iztu/files/wandb-metadata.json

@@ -0,0 +1,35 @@
+{
+  "os": "Linux-5.15.0-1040-nvidia-x86_64-with-glibc2.35",
+  "python": "CPython 3.11.11",
+  "startedAt": "2025-03-28T16:54:25.328152Z",
+  "args": [
+    "--local_rank=0",
+    "--model_name_or_path",
+    "/aifs4su/yaodong/hantao/models/llava-v1.6-mistral-7b-hf",
+    "--train_datasets",
+    "/aifs4su/yaodong/hantao/datasets/MMInstruct-GPT4V_mistral-7b_cosi_cut/merged/top1-10",
+    "--train_template",
+    "MM_TI2T_LLAVA",
+    "--train_split",
+    "train",
+    "--train_name",
+    "text-image-to-text",
+    "--output_dir",
+    "../outputs/test_7B",
+    "--save_total_limit",
+    "6",
+    "--train_batch_size",
+    "8",
+    "--epochs",
+    "3"
+  ],
+  "program": "-m align_anything.trainers.text_image_to_text.sft",
+  "git": {
+    "remote": "git@github.com-hantao:PKU-Alignment/align-anything.git",
+    "commit": "106588f9802757a3283c1aff1f33ea9afd737f31"
+  },
+  "email": "2200017789@stu.pku.edu.cn",
+  "root": "../outputs/test_7B",
+  "host": "dgx-092",
+  "executable": "/aifs4su/yaodong/miniconda3/envs/hantao_llama/bin/python"
+}

run-20250329_005425-3al6iztu/logs/debug-core.log

@@ -0,0 +1,7 @@
+{"time":"2025-03-29T00:54:24.691982358+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpszvpq4wi/port-48756.txt","pid":48756,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false}
+{"time":"2025-03-29T00:54:24.692861871+08:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":48756}
+{"time":"2025-03-29T00:54:24.692845205+08:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":42415,"Zone":""}}
+{"time":"2025-03-29T00:54:24.873664584+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:37608"}
+{"time":"2025-03-29T00:54:25.329808834+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"3al6iztu","id":"127.0.0.1:37608"}
+{"time":"2025-03-29T00:54:25.545982861+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"3al6iztu","id":"127.0.0.1:37608"}
+{"time":"2025-03-29T00:54:26.490223915+08:00","level":"INFO","msg":"received shutdown signal","signal":15}

run-20250329_005425-3al6iztu/logs/debug-internal.log

@@ -0,0 +1,8 @@
+{"time":"2025-03-29T00:54:25.331350097+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"../outputs/test_7B/wandb/run-20250329_005425-3al6iztu/logs/debug-core.log"}
+{"time":"2025-03-29T00:54:25.545872725+08:00","level":"INFO","msg":"created new stream","id":"3al6iztu"}
+{"time":"2025-03-29T00:54:25.54597415+08:00","level":"INFO","msg":"stream: started","id":"3al6iztu"}
+{"time":"2025-03-29T00:54:25.546010693+08:00","level":"INFO","msg":"sender: started","stream_id":"3al6iztu"}
+{"time":"2025-03-29T00:54:25.546003597+08:00","level":"INFO","msg":"handler: started","stream_id":"3al6iztu"}
+{"time":"2025-03-29T00:54:25.546055332+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"3al6iztu"}
+{"time":"2025-03-29T00:54:25.844887265+08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2025-03-29T00:54:26.062125748+08:00","level":"INFO","msg":"Stopping system monitor"}

run-20250329_005425-3al6iztu/logs/debug.log

@@ -0,0 +1,26 @@
+2025-03-29 00:54:25,324 INFO    MainThread:48756 [wandb_setup.py:_flush():67] Current SDK version is 0.19.8
+2025-03-29 00:54:25,325 INFO    MainThread:48756 [wandb_setup.py:_flush():67] Configure stats pid to 48756
+2025-03-29 00:54:25,325 INFO    MainThread:48756 [wandb_setup.py:_flush():67] Loading settings from /home/yangyaodong/.config/wandb/settings
+2025-03-29 00:54:25,325 INFO    MainThread:48756 [wandb_setup.py:_flush():67] Loading settings from /aifs4su/yaodong/hantao/align-anything/scripts/wandb/settings
+2025-03-29 00:54:25,325 INFO    MainThread:48756 [wandb_setup.py:_flush():67] Loading settings from environment variables
+2025-03-29 00:54:25,325 INFO    MainThread:48756 [wandb_init.py:setup_run_log_directory():647] Logging user logs to ../outputs/test_7B/wandb/run-20250329_005425-3al6iztu/logs/debug.log
+2025-03-29 00:54:25,325 INFO    MainThread:48756 [wandb_init.py:setup_run_log_directory():648] Logging internal logs to ../outputs/test_7B/wandb/run-20250329_005425-3al6iztu/logs/debug-internal.log
+2025-03-29 00:54:25,325 INFO    MainThread:48756 [wandb_init.py:init():761] calling init triggers
+2025-03-29 00:54:25,325 INFO    MainThread:48756 [wandb_init.py:init():766] wandb.init called with sweep_config: {}
+config: {'train_cfgs': {'save_checkpoint': False, 'load_checkpoint': False, 'ds_cfgs': 'ds_z3_config.json', 'epochs': 3, 'seed': 42, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 1, 'gradient_accumulation_steps': 16, 'gradient_checkpointing': True, 'learning_rate': 2e-05, 'lr_scheduler_type': 'cosine', 'lr_warmup_ratio': 0.03, 'weight_decay': 0.0, 'adam_betas': [0.9, 0.95], 'adam_epsilon': 1e-08, 'bf16': True, 'fp16': False, 'eval_strategy': 'epoch', 'eval_interval': 10, 'freeze_mm_proj': False, 'freeze_vision_tower': True, 'freeze_language_model': False, 'max_grad_norm': 1.0}, 'data_cfgs': {'load_multi_datasets': False, 'train_datasets': '/aifs4su/yaodong/hantao/datasets/MMInstruct-GPT4V_mistral-7b_cosi_cut/merged/top1-10', 'train_template': 'MM_TI2T_LLAVA', 'train_size': {}, 'train_split': 'train', 'train_name': 'text-image-to-text', 'train_data_files': {}, 'train_optional_args': [], 'eval_datasets': {}, 'eval_template': {}, 'eval_name': {}, 'eval_size': {}, 'eval_split': {}, 'eval_subset': {}, 'eval_data_files': {}, 'eval_optional_args': []}, 'logger_cfgs': {'log_type': 'wandb', 'log_project': 'align-anything', 'log_run_name': 'sft', 'output_dir': '../outputs/test_7B', 'cache_dir': {}, 'save_total_limit': 6}, 'model_cfgs': {'model_name_or_path': '/aifs4su/yaodong/hantao/models/llava-v1.6-mistral-7b-hf', 'trust_remote_code': True, 'model_max_length': 2048}, 'special_tokens': {}, '_wandb': {}}
+2025-03-29 00:54:25,325 INFO    MainThread:48756 [wandb_init.py:init():784] starting backend
+2025-03-29 00:54:25,325 INFO    MainThread:48756 [wandb_init.py:init():788] sending inform_init request
+2025-03-29 00:54:25,327 INFO    MainThread:48756 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-03-29 00:54:25,328 INFO    MainThread:48756 [wandb_init.py:init():798] backend started and connected
+2025-03-29 00:54:25,329 INFO    MainThread:48756 [wandb_init.py:init():891] updated telemetry
+2025-03-29 00:54:25,340 INFO    MainThread:48756 [wandb_init.py:init():915] communicating run to backend with 90.0 second timeout
+2025-03-29 00:54:25,842 INFO    MainThread:48756 [wandb_init.py:init():990] starting run threads in backend
+2025-03-29 00:54:26,016 INFO    MainThread:48756 [wandb_run.py:_console_start():2375] atexit reg
+2025-03-29 00:54:26,016 INFO    MainThread:48756 [wandb_run.py:_redirect():2227] redirect: wrap_raw
+2025-03-29 00:54:26,016 INFO    MainThread:48756 [wandb_run.py:_redirect():2292] Wrapping output streams.
+2025-03-29 00:54:26,016 INFO    MainThread:48756 [wandb_run.py:_redirect():2315] Redirects installed.
+2025-03-29 00:54:26,019 INFO    MainThread:48756 [wandb_init.py:init():1032] run started, returning control to user process
+2025-03-29 00:54:26,029 INFO    MainThread:48756 [wandb_run.py:_finish():2112] finishing run htlou/align-anything/3al6iztu
+2025-03-29 00:54:26,030 INFO    MainThread:48756 [wandb_run.py:_atexit_cleanup():2340] got exitcode: 0
+2025-03-29 00:54:26,030 INFO    MainThread:48756 [wandb_run.py:_restore():2322] restore
+2025-03-29 00:54:26,030 INFO    MainThread:48756 [wandb_run.py:_restore():2328] restore done

run-20250329_005541-bq1jaffa/files/config.yaml

@@ -0,0 +1,95 @@
+_wandb:
+    value:
+        cli_version: 0.19.8
+        m: []
+        python_version: 3.11.11
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 41
+                - 49
+                - 51
+                - 53
+                - 55
+                - 63
+                - 71
+                - 83
+                - 98
+                - 105
+            "2":
+                - 1
+                - 5
+                - 11
+                - 41
+                - 49
+                - 51
+                - 53
+                - 55
+                - 63
+                - 71
+                - 83
+                - 98
+                - 105
+            "3":
+                - 2
+                - 13
+                - 16
+                - 23
+                - 55
+            "4": 3.11.11
+            "5": 0.19.8
+            "6": 4.51.0.dev0
+            "8":
+                - 5
+            "12": 0.19.8
+            "13": linux-x86_64
+data_cfgs:
+    value:
+        eval_optional_args: []
+        load_multi_datasets: false
+        train_datasets: /aifs4su/yaodong/hantao/datasets/MMInstruct-GPT4V_mistral-7b_cosi_cut/merged/top1-10
+        train_name: text-image-to-text
+        train_optional_args: []
+        train_split: train
+        train_template: MM_TI2T_LLAVA
+logger_cfgs:
+    value:
+        log_project: align-anything
+        log_run_name: sft
+        log_type: wandb
+        output_dir: ../outputs/test_7B
+        save_total_limit: 6
+model_cfgs:
+    value:
+        model_max_length: 2048
+        model_name_or_path: /aifs4su/yaodong/hantao/models/llava-v1.6-mistral-7b-hf
+        trust_remote_code: true
+train_cfgs:
+    value:
+        adam_betas:
+            - 0.9
+            - 0.95
+        adam_epsilon: 1e-08
+        bf16: true
+        ds_cfgs: ds_z3_config.json
+        epochs: 3
+        eval_interval: 10
+        eval_strategy: epoch
+        fp16: false
+        freeze_language_model: false
+        freeze_mm_proj: false
+        freeze_vision_tower: true
+        gradient_accumulation_steps: 16
+        gradient_checkpointing: true
+        learning_rate: 2e-05
+        load_checkpoint: false
+        lr_scheduler_type: cosine
+        lr_warmup_ratio: 0.03
+        max_grad_norm: 1
+        per_device_eval_batch_size: 1
+        per_device_train_batch_size: 1
+        save_checkpoint: false
+        seed: 42
+        weight_decay: 0

run-20250329_005541-bq1jaffa/files/output.log

@@ -0,0 +1,22 @@
+***** Running training *****
+Resuming from checkpoint 3/3 epoch :   0%|                                                                                                                                                                                                                                                                    | 0/7326 [00:00<?, ?it/s]
+Train dataloader:  <torch.utils.data.dataloader.DataLoader object at 0x155108341d10>
+<enumerate object at 0x154ff4f2b6f0>
+Check if empty:  False
+First data:  {'id': 182941, 'image': 'ocr/0001/00000300.jpg', 'conversations': [{'from': 'human', 'value': '<image>\nDo you have the ability to recognize and translate textual information in images?\nA. The image shows the front cover of a book titled "THE CALMING COLLECTION PRESENTS GOODBYE WORRIES TRAIN YOUR MIND TO QUIET YOUR THOUGHTS ANYTIME."\nB. The image displays the product label of a herbal tea called "THE CALMING COLLECTION PRESENTS GOODBYE WORRIES TRAIN YOUR MIND TO QUIET YOUR THOUGHTS ANYTIME."\nC. The image represents a poster promoting a workshop on "THE CALMING COLLECTION PRESENTS GOODBYE WORRIES TRAIN YOUR MIND TO QUIET YOUR THOUGHTS ANYTIME."\nD.  "THE CALMING COLLECTION PRESENTS GOODBYE WORRIES TRAIN YOUR MIND TO QUIET YOUR THOUGHTS ANYTIME."\nAnswer with the option\'s letter from the given choices directly.'}, {'from': 'gpt', 'value': 'D'}], 'cosi': 70.53595100308407}
+Train dataloader:  <torch.utils.data.dataloader.DataLoader object at 0x155108341d10>
+<enumerate object at 0x155005bebd80>
+Check if empty:  False
+First data:  {'id': 182941, 'image': 'ocr/0001/00000300.jpg', 'conversations': [{'from': 'human', 'value': '<image>\nDo you have the ability to recognize and translate textual information in images?\nA. The image shows the front cover of a book titled "THE CALMING COLLECTION PRESENTS GOODBYE WORRIES TRAIN YOUR MIND TO QUIET YOUR THOUGHTS ANYTIME."\nB. The image displays the product label of a herbal tea called "THE CALMING COLLECTION PRESENTS GOODBYE WORRIES TRAIN YOUR MIND TO QUIET YOUR THOUGHTS ANYTIME."\nC. The image represents a poster promoting a workshop on "THE CALMING COLLECTION PRESENTS GOODBYE WORRIES TRAIN YOUR MIND TO QUIET YOUR THOUGHTS ANYTIME."\nD.  "THE CALMING COLLECTION PRESENTS GOODBYE WORRIES TRAIN YOUR MIND TO QUIET YOUR THOUGHTS ANYTIME."\nAnswer with the option\'s letter from the given choices directly.'}, {'from': 'gpt', 'value': 'D'}], 'cosi': 70.53595100308407}
+Train dataloader:  <torch.utils.data.dataloader.DataLoader object at 0x155108341d10>
+<enumerate object at 0x154ff4f75080>
+Check if empty:  False
+First data:  {'id': 182941, 'image': 'ocr/0001/00000300.jpg', 'conversations': [{'from': 'human', 'value': '<image>\nDo you have the ability to recognize and translate textual information in images?\nA. The image shows the front cover of a book titled "THE CALMING COLLECTION PRESENTS GOODBYE WORRIES TRAIN YOUR MIND TO QUIET YOUR THOUGHTS ANYTIME."\nB. The image displays the product label of a herbal tea called "THE CALMING COLLECTION PRESENTS GOODBYE WORRIES TRAIN YOUR MIND TO QUIET YOUR THOUGHTS ANYTIME."\nC. The image represents a poster promoting a workshop on "THE CALMING COLLECTION PRESENTS GOODBYE WORRIES TRAIN YOUR MIND TO QUIET YOUR THOUGHTS ANYTIME."\nD.  "THE CALMING COLLECTION PRESENTS GOODBYE WORRIES TRAIN YOUR MIND TO QUIET YOUR THOUGHTS ANYTIME."\nAnswer with the option\'s letter from the given choices directly.'}, {'from': 'gpt', 'value': 'D'}], 'cosi': 70.53595100308407}
+Saving model to "../outputs/test_7B/slice_end" ...
+Saving 16-bit model...
+[2025-03-29 00:55:50,235] [INFO] [logging.py:107:log_dist] [Rank 0] [Torch] Checkpoint global_step0 is about to be saved!
+[2025-03-29 00:55:50,236] [INFO] [engine.py:3831:save_16bit_model] Saving model weights to ../outputs/test_7B/slice_end/pytorch_model.bin, tag: global_step0
+[2025-03-29 00:55:50,236] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ../outputs/test_7B/slice_end/pytorch_model.bin...
+[2025-03-29 00:56:05,543] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ../outputs/test_7B/slice_end/pytorch_model.bin.
+[2025-03-29 00:56:05,544] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step0 is ready now!
+Model saved!

run-20250329_005541-bq1jaffa/files/requirements.txt

@@ -0,0 +1,167 @@
+maskrcnn_benchmark==0.0.0
+webdataset==0.2.111
+websockets==15.0.1
+typer==0.15.2
+blobfile==3.0.0
+pooch==1.8.2
+python-dateutil==2.9.0.post0
+gmpy2==2.2.1
+httpcore==1.0.7
+charset-normalizer==3.3.2
+torchlibrosa==0.1.0
+multiprocess==0.70.16
+Werkzeug==3.1.3
+aiofiles==23.2.1
+six==1.17.0
+typing_extensions==4.12.2
+psutil==7.0.0
+frozenlist==1.5.0
+einops==0.8.1
+flash_attn==2.7.4.post1
+PySocks==1.7.1
+regex==2024.11.6
+markdown-it-py==3.0.0
+ruff==0.11.2
+docker-pycreds==0.4.0
+protobuf==5.29.4
+resampy==0.4.3
+aiohappyeyeballs==2.6.1
+httpx==0.28.1
+encodec==0.1.1
+ffmpy==0.5.0
+mkl_random==1.2.8
+soxr==0.5.0.post1
+absl-py==2.2.1
+networkx==3.4.2
+h5py==3.13.0
+hjson==3.1.0
+tensorboard==2.19.0
+aiosignal==1.3.2
+pip==25.0
+triton==3.1.0
+zipp==3.21.0
+ftfy==6.3.1
+attrs==25.3.0
+requests==2.32.3
+progressbar==2.5
+sniffio==1.3.1
+lxml==5.3.1
+starlette==0.46.1
+Markdown==3.7
+mdurl==0.1.2
+torchaudio==2.5.1
+safetensors==0.5.3
+opencv-python==4.6.0.66
+torchvision==0.20.1
+shellingham==1.5.4
+gradio==5.23.1
+timm==1.0.15
+multidict==6.2.0
+semantic-version==2.10.0
+numba==0.60.0
+gradio_client==1.8.0
+pydantic_core==2.33.0
+dill==0.3.8
+msgpack==1.1.0
+sentry-sdk==2.24.1
+grpcio==1.71.0
+cffi==1.17.1
+PyYAML==6.0.2
+tensorboard-data-server==0.7.2
+fastapi==0.115.12
+lazy_loader==0.4
+mkl_fft==1.3.11
+annotated-types==0.7.0
+scikit-learn==1.6.1
+wget==3.2
+setuptools==75.8.0
+certifi==2025.1.31
+click==8.1.8
+laion_clap==1.1.5
+Pygments==2.19.1
+tomlkit==0.13.2
+idna==3.7
+propcache==0.3.1
+platformdirs==4.3.7
+align-anything==0.0.1.dev0
+deepspeed==0.16.5
+smmap==5.0.2
+pillow==11.1.0
+typing-inspection==0.4.0
+braceexpand==0.1.7
+decorator==5.2.1
+pandas==2.2.3
+huggingface-hub==0.29.3
+pyarrow==19.0.1
+tokenizers==0.21.1
+GitPython==3.1.44
+xxhash==3.5.0
+packaging==24.2
+numpy==1.23.4
+setproctitle==1.3.5
+llvmlite==0.43.0
+tiktoken==0.9.0
+mpmath==1.3.0
+nvidia-ml-py==12.570.86
+pydantic==2.11.0
+datasets==3.5.0
+librosa==0.11.0
+frechet_audio_distance==0.3.1
+sympy==1.13.1
+safehttpx==0.1.6
+Jinja2==3.1.6
+h11==0.14.0
+aiohttp==3.11.14
+diffusers==0.32.2
+tqdm==4.67.1
+filelock==3.13.1
+transformers==4.51.0.dev0
+scipy==1.10.1
+audioread==3.0.1
+sentencepiece==0.2.0
+pytz==2025.2
+tzdata==2025.2
+python-multipart==0.0.20
+urllib3==2.3.0
+pycryptodomex==3.22.0
+yarl==1.18.3
+pydub==0.25.1
+pycparser==2.22
+soundfile==0.13.1
+wcwidth==0.2.13
+groovy==0.1.2
+torch==2.5.1
+anyio==4.9.0
+wandb==0.19.8
+joblib==1.4.2
+fsspec==2024.12.0
+peft==0.15.1
+accelerate==1.5.2
+py-cpuinfo==9.0.0
+uvicorn==0.34.0
+orjson==3.10.16
+Brotli==1.0.9
+rich==13.9.4
+importlib_metadata==8.6.1
+ninja==1.11.1.4
+wheel==0.45.1
+MarkupSafe==3.0.2
+threadpoolctl==3.6.0
+gitdb==4.0.12
+mkl-service==2.4.0
+typing_extensions==4.12.2
+tomli==2.0.1
+zipp==3.19.2
+wheel==0.43.0
+jaraco.text==3.12.1
+packaging==24.2
+autocommand==2.2.2
+jaraco.functools==4.0.1
+jaraco.collections==5.1.0
+platformdirs==4.2.2
+more-itertools==10.3.0
+inflect==7.3.1
+jaraco.context==5.3.0
+typeguard==4.3.0
+backports.tarfile==1.2.0
+importlib_metadata==8.0.0

run-20250329_005541-bq1jaffa/files/wandb-metadata.json

@@ -0,0 +1,106 @@
+{
+  "os": "Linux-5.15.0-1040-nvidia-x86_64-with-glibc2.35",
+  "python": "CPython 3.11.11",
+  "startedAt": "2025-03-28T16:55:41.711696Z",
+  "args": [
+    "--local_rank=0",
+    "--model_name_or_path",
+    "/aifs4su/yaodong/hantao/models/llava-v1.6-mistral-7b-hf",
+    "--train_datasets",
+    "/aifs4su/yaodong/hantao/datasets/MMInstruct-GPT4V_mistral-7b_cosi_cut/merged/top1-10",
+    "--train_template",
+    "MM_TI2T_LLAVA",
+    "--train_split",
+    "train",
+    "--train_name",
+    "text-image-to-text",
+    "--output_dir",
+    "../outputs/test_7B",
+    "--save_total_limit",
+    "6",
+    "--train_batch_size",
+    "8",
+    "--epochs",
+    "3"
+  ],
+  "program": "-m align_anything.trainers.text_image_to_text.sft",
+  "git": {
+    "remote": "git@github.com-hantao:PKU-Alignment/align-anything.git",
+    "commit": "106588f9802757a3283c1aff1f33ea9afd737f31"
+  },
+  "email": "2200017789@stu.pku.edu.cn",
+  "root": "../outputs/test_7B",
+  "host": "dgx-092",
+  "executable": "/aifs4su/yaodong/miniconda3/envs/hantao_llama/bin/python",
+  "cpu_count": 112,
+  "cpu_count_logical": 224,
+  "gpu": "NVIDIA H800",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1888556142592",
+      "used": "148609179648"
+    }
+  },
+  "memory": {
+    "total": "2164195454976"
+  },
+  "cpu": {
+    "count": 112,
+    "countLogical": 224
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA H800",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H800",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H800",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H800",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H800",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H800",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H800",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H800",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    }
+  ],
+  "slurm": {
+    "conf": "/cm/shared/apps/slurm/var/etc/slurm/slurm.conf"
+  },
+  "cudaVersion": "12.2"
+}

run-20250329_005541-bq1jaffa/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":24}}

run-20250329_005541-bq1jaffa/logs/debug-core.log

@@ -0,0 +1,15 @@
+{"time":"2025-03-29T00:55:41.128572776+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpaadyf_35/port-52806.txt","pid":52806,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false}
+{"time":"2025-03-29T00:55:41.129538675+08:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":52806}
+{"time":"2025-03-29T00:55:41.129533087+08:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":38349,"Zone":""}}
+{"time":"2025-03-29T00:55:41.310059156+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:46506"}
+{"time":"2025-03-29T00:55:41.713050692+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"bq1jaffa","id":"127.0.0.1:46506"}
+{"time":"2025-03-29T00:55:41.928489719+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"bq1jaffa","id":"127.0.0.1:46506"}
+{"time":"2025-03-29T00:56:07.1453019+08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"bq1jaffa","id":"127.0.0.1:46506"}
+{"time":"2025-03-29T00:56:07.146579952+08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"bq1jaffa","id":"127.0.0.1:46506"}
+{"time":"2025-03-29T00:56:08.145760813+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:46506"}
+{"time":"2025-03-29T00:56:08.145785955+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:46506"}
+{"time":"2025-03-29T00:56:08.14579164+08:00","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-03-29T00:56:08.145814767+08:00","level":"INFO","msg":"connection: closing","id":"127.0.0.1:46506"}
+{"time":"2025-03-29T00:56:08.145848309+08:00","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:46506"}
+{"time":"2025-03-29T00:56:08.145850825+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:46506"}
+{"time":"2025-03-29T00:56:08.145853752+08:00","level":"INFO","msg":"server is closed"}

run-20250329_005541-bq1jaffa/logs/debug-internal.log

@@ -0,0 +1,16 @@
+{"time":"2025-03-29T00:55:41.714661009+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"../outputs/test_7B/wandb/run-20250329_005541-bq1jaffa/logs/debug-core.log"}
+{"time":"2025-03-29T00:55:41.928417193+08:00","level":"INFO","msg":"created new stream","id":"bq1jaffa"}
+{"time":"2025-03-29T00:55:41.928482583+08:00","level":"INFO","msg":"stream: started","id":"bq1jaffa"}
+{"time":"2025-03-29T00:55:41.928501227+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"bq1jaffa"}
+{"time":"2025-03-29T00:55:41.928513756+08:00","level":"INFO","msg":"sender: started","stream_id":"bq1jaffa"}
+{"time":"2025-03-29T00:55:41.928511582+08:00","level":"INFO","msg":"handler: started","stream_id":"bq1jaffa"}
+{"time":"2025-03-29T00:55:42.22838417+08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2025-03-29T00:56:06.166942314+08:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2025-03-29T00:56:06.16760159+08:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2025-03-29T00:56:06.921670341+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-03-29T00:56:07.143514035+08:00","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2025-03-29T00:56:07.145585911+08:00","level":"INFO","msg":"stream: closing","id":"bq1jaffa"}
+{"time":"2025-03-29T00:56:07.145626123+08:00","level":"INFO","msg":"handler: closed","stream_id":"bq1jaffa"}
+{"time":"2025-03-29T00:56:07.145635477+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"bq1jaffa"}
+{"time":"2025-03-29T00:56:07.145639618+08:00","level":"INFO","msg":"sender: closed","stream_id":"bq1jaffa"}
+{"time":"2025-03-29T00:56:07.146568555+08:00","level":"INFO","msg":"stream: closed","id":"bq1jaffa"}

run-20250329_005541-bq1jaffa/logs/debug.log

@@ -0,0 +1,29 @@
+2025-03-29 00:55:41,708 INFO    MainThread:52806 [wandb_setup.py:_flush():67] Current SDK version is 0.19.8
+2025-03-29 00:55:41,709 INFO    MainThread:52806 [wandb_setup.py:_flush():67] Configure stats pid to 52806
+2025-03-29 00:55:41,709 INFO    MainThread:52806 [wandb_setup.py:_flush():67] Loading settings from /home/yangyaodong/.config/wandb/settings
+2025-03-29 00:55:41,709 INFO    MainThread:52806 [wandb_setup.py:_flush():67] Loading settings from /aifs4su/yaodong/hantao/align-anything/scripts/wandb/settings
+2025-03-29 00:55:41,709 INFO    MainThread:52806 [wandb_setup.py:_flush():67] Loading settings from environment variables
+2025-03-29 00:55:41,709 INFO    MainThread:52806 [wandb_init.py:setup_run_log_directory():647] Logging user logs to ../outputs/test_7B/wandb/run-20250329_005541-bq1jaffa/logs/debug.log
+2025-03-29 00:55:41,709 INFO    MainThread:52806 [wandb_init.py:setup_run_log_directory():648] Logging internal logs to ../outputs/test_7B/wandb/run-20250329_005541-bq1jaffa/logs/debug-internal.log
+2025-03-29 00:55:41,709 INFO    MainThread:52806 [wandb_init.py:init():761] calling init triggers
+2025-03-29 00:55:41,709 INFO    MainThread:52806 [wandb_init.py:init():766] wandb.init called with sweep_config: {}
+config: {'train_cfgs': {'save_checkpoint': False, 'load_checkpoint': False, 'ds_cfgs': 'ds_z3_config.json', 'epochs': 3, 'seed': 42, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 1, 'gradient_accumulation_steps': 16, 'gradient_checkpointing': True, 'learning_rate': 2e-05, 'lr_scheduler_type': 'cosine', 'lr_warmup_ratio': 0.03, 'weight_decay': 0.0, 'adam_betas': [0.9, 0.95], 'adam_epsilon': 1e-08, 'bf16': True, 'fp16': False, 'eval_strategy': 'epoch', 'eval_interval': 10, 'freeze_mm_proj': False, 'freeze_vision_tower': True, 'freeze_language_model': False, 'max_grad_norm': 1.0}, 'data_cfgs': {'load_multi_datasets': False, 'train_datasets': '/aifs4su/yaodong/hantao/datasets/MMInstruct-GPT4V_mistral-7b_cosi_cut/merged/top1-10', 'train_template': 'MM_TI2T_LLAVA', 'train_size': {}, 'train_split': 'train', 'train_name': 'text-image-to-text', 'train_data_files': {}, 'train_optional_args': [], 'eval_datasets': {}, 'eval_template': {}, 'eval_name': {}, 'eval_size': {}, 'eval_split': {}, 'eval_subset': {}, 'eval_data_files': {}, 'eval_optional_args': []}, 'logger_cfgs': {'log_type': 'wandb', 'log_project': 'align-anything', 'log_run_name': 'sft', 'output_dir': '../outputs/test_7B', 'cache_dir': {}, 'save_total_limit': 6}, 'model_cfgs': {'model_name_or_path': '/aifs4su/yaodong/hantao/models/llava-v1.6-mistral-7b-hf', 'trust_remote_code': True, 'model_max_length': 2048}, 'special_tokens': {}, '_wandb': {}}
+2025-03-29 00:55:41,709 INFO    MainThread:52806 [wandb_init.py:init():784] starting backend
+2025-03-29 00:55:41,709 INFO    MainThread:52806 [wandb_init.py:init():788] sending inform_init request
+2025-03-29 00:55:41,711 INFO    MainThread:52806 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-03-29 00:55:41,711 INFO    MainThread:52806 [wandb_init.py:init():798] backend started and connected
+2025-03-29 00:55:41,712 INFO    MainThread:52806 [wandb_init.py:init():891] updated telemetry
+2025-03-29 00:55:41,722 INFO    MainThread:52806 [wandb_init.py:init():915] communicating run to backend with 90.0 second timeout
+2025-03-29 00:55:42,226 INFO    MainThread:52806 [wandb_init.py:init():990] starting run threads in backend
+2025-03-29 00:55:42,378 INFO    MainThread:52806 [wandb_run.py:_console_start():2375] atexit reg
+2025-03-29 00:55:42,378 INFO    MainThread:52806 [wandb_run.py:_redirect():2227] redirect: wrap_raw
+2025-03-29 00:55:42,378 INFO    MainThread:52806 [wandb_run.py:_redirect():2292] Wrapping output streams.
+2025-03-29 00:55:42,378 INFO    MainThread:52806 [wandb_run.py:_redirect():2315] Redirects installed.
+2025-03-29 00:55:42,380 INFO    MainThread:52806 [wandb_init.py:init():1032] run started, returning control to user process
+2025-03-29 00:56:06,165 INFO    MainThread:52806 [wandb_run.py:_finish():2112] finishing run htlou/align-anything/bq1jaffa
+2025-03-29 00:56:06,165 INFO    MainThread:52806 [wandb_run.py:_atexit_cleanup():2340] got exitcode: 0
+2025-03-29 00:56:06,166 INFO    MainThread:52806 [wandb_run.py:_restore():2322] restore
+2025-03-29 00:56:06,166 INFO    MainThread:52806 [wandb_run.py:_restore():2328] restore done
+2025-03-29 00:56:07,144 INFO    MainThread:52806 [wandb_run.py:_footer_history_summary_info():3956] rendering history
+2025-03-29 00:56:07,144 INFO    MainThread:52806 [wandb_run.py:_footer_history_summary_info():3988] rendering summary
+2025-03-29 00:56:07,144 INFO    MainThread:52806 [wandb_run.py:_footer_sync_info():3917] logging synced files