Add files using upload-large-folder tool

.gitattributes

@@ -36,3 +36,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 all_checkpoints/stage1_06290009_deepspeed/wandb/run-20250629_001200-vgvxxzqc/run-vgvxxzqc.wandb filter=lfs diff=lfs merge=lfs -text
 all_checkpoints/stage1_07041727_2dataset/wandb/run-20250704_172854-6bkqzmou/run-6bkqzmou.wandb filter=lfs diff=lfs merge=lfs -text
 all_checkpoints/stage2_07070337_2datasets_noconstruct/wandb/run-20250707_041231-gtrtcbb9/run-gtrtcbb9.wandb filter=lfs diff=lfs merge=lfs -text
+all_checkpoints/stage2_07070513_2datasets_construct/wandb/run-20250707_052104-615z4bme/run-615z4bme.wandb filter=lfs diff=lfs merge=lfs -text
+all_checkpoints/stage2_07070513_2datasets_construct/wandb/run-20250707_053222-9cjzn0v3/run-9cjzn0v3.wandb filter=lfs diff=lfs merge=lfs -text
+all_checkpoints/stage2_07301646_2datasets_construct/wandb/run-20250730_175623-pbf2bxo6/run-pbf2bxo6.wandb filter=lfs diff=lfs merge=lfs -text

all_checkpoints/stage2_07070513_2datasets_construct/epoch=09.ckpt/checkpoint/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab1a6ea1ac55ce9616532e761371379f0cb306bbf194e29d1a44cabe01dd4e3a
+size 156403632

all_checkpoints/stage2_07070513_2datasets_construct/epoch=09.ckpt/checkpoint/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c0aa50226009f0d8a6a23d3d6e98e194a4f2a944d95a7f46bcde8d3aebd98e5
+size 156402992

all_checkpoints/stage2_07070513_2datasets_construct/epoch=09.ckpt/checkpoint/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd423ebd707139e3d47d635ea2251cb7f655b5f24650085a7abf71a1f26f3d05
+size 156403376

all_checkpoints/stage2_07070513_2datasets_construct/epoch=09.ckpt/checkpoint/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:850b33a5acf35f5909aae8ad5672d7895c82f2024dc5b08d95b9ff338e08e519
+size 156403120

all_checkpoints/stage2_07070513_2datasets_construct/epoch=09.ckpt/checkpoint/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:942855deca23d990f0db7f0ce36aafba768e914d42db3b7f38810250bf9e2c25
+size 156402416

all_checkpoints/stage2_07070513_2datasets_construct/epoch=09.ckpt/checkpoint/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f14691a96f4b8d119ff12b6282ae802455fbec50f06f044bd597970f8331aca5
+size 156403696

all_checkpoints/stage2_07070513_2datasets_construct/epoch=09.ckpt/checkpoint/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:60586788404b4008997d9fc2cee3b07a9a3f28324c0b0390bf54ef165acd2580
+size 156402992

all_checkpoints/stage2_07070513_2datasets_construct/epoch=09.ckpt/checkpoint/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b17dec7f7d4ac6aa06b05dfc462965b64f54028ed539e422ac2695da0c87828
+size 156417904

all_checkpoints/stage2_07070513_2datasets_construct/epoch=09.ckpt/checkpoint/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dbc87c01c9bbe626db3b00ce1ecc91c91503d810e852467d60441c897aa405eb
+size 208795192

all_checkpoints/stage2_07070513_2datasets_construct/epoch=09.ckpt/converted.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb1faf00f4930709a0ce83e6276411772ea0ce8357a6dde45856eb25d7ba33b7
+size 417200356

all_checkpoints/stage2_07070513_2datasets_construct/last.ckpt/checkpoint/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab1a6ea1ac55ce9616532e761371379f0cb306bbf194e29d1a44cabe01dd4e3a
+size 156403632

all_checkpoints/stage2_07070513_2datasets_construct/last.ckpt/checkpoint/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c0aa50226009f0d8a6a23d3d6e98e194a4f2a944d95a7f46bcde8d3aebd98e5
+size 156402992

all_checkpoints/stage2_07070513_2datasets_construct/last.ckpt/checkpoint/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd423ebd707139e3d47d635ea2251cb7f655b5f24650085a7abf71a1f26f3d05
+size 156403376

all_checkpoints/stage2_07070513_2datasets_construct/last.ckpt/checkpoint/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:850b33a5acf35f5909aae8ad5672d7895c82f2024dc5b08d95b9ff338e08e519
+size 156403120

all_checkpoints/stage2_07070513_2datasets_construct/last.ckpt/checkpoint/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:942855deca23d990f0db7f0ce36aafba768e914d42db3b7f38810250bf9e2c25
+size 156402416

all_checkpoints/stage2_07070513_2datasets_construct/last.ckpt/checkpoint/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f14691a96f4b8d119ff12b6282ae802455fbec50f06f044bd597970f8331aca5
+size 156403696

all_checkpoints/stage2_07070513_2datasets_construct/last.ckpt/checkpoint/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:60586788404b4008997d9fc2cee3b07a9a3f28324c0b0390bf54ef165acd2580
+size 156402992

all_checkpoints/stage2_07070513_2datasets_construct/last.ckpt/checkpoint/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b17dec7f7d4ac6aa06b05dfc462965b64f54028ed539e422ac2695da0c87828
+size 156417904

all_checkpoints/stage2_07070513_2datasets_construct/last.ckpt/checkpoint/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dbc87c01c9bbe626db3b00ce1ecc91c91503d810e852467d60441c897aa405eb
+size 208795192

all_checkpoints/stage2_07070513_2datasets_construct/last.ckpt/latest

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47320987f9a49d5b00119b960f247a956773f57543982b8bfcb6da5bb3afd9ef
+size 10

all_checkpoints/stage2_07070513_2datasets_construct/last.ckpt/zero_to_fp32.py

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:46497565ccf2b4a8b1f6f18c8341042f3749605a94335c81f69df1bd268af64f
+size 33272

all_checkpoints/stage2_07070513_2datasets_construct/wandb/debug-internal.log

@@ -0,0 +1,29 @@
+{"time":"2025-07-07T05:32:22.544190733+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.11","symlink path":"all_checkpoints/stage2_07070513_2datasets_construct/wandb/run-20250707_053222-9cjzn0v3/logs/debug-core.log"}
+{"time":"2025-07-07T05:32:23.597067843+08:00","level":"INFO","msg":"created new stream","id":"9cjzn0v3"}
+{"time":"2025-07-07T05:32:23.59711309+08:00","level":"INFO","msg":"stream: started","id":"9cjzn0v3"}
+{"time":"2025-07-07T05:32:23.59715533+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"9cjzn0v3"}
+{"time":"2025-07-07T05:32:23.597176058+08:00","level":"INFO","msg":"handler: started","stream_id":"9cjzn0v3"}
+{"time":"2025-07-07T05:32:23.597249736+08:00","level":"INFO","msg":"sender: started","stream_id":"9cjzn0v3"}
+{"time":"2025-07-07T05:32:24.815832776+08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2025-07-07T16:23:26.191588391+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07070513_2datasets_construct/9cjzn0v3/file_stream\": read tcp 10.1.2.136:46082->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-08T07:36:01.662714436+08:00","level":"INFO","msg":"api: retrying HTTP error","status":504,"url":"https://api.bandw.top/files/gia0603yucca/stage2_07070513_2datasets_construct/9cjzn0v3/file_stream","body":"error code: 504"}
+{"time":"2025-07-08T07:39:35.510926561+08:00","level":"INFO","msg":"api: retrying HTTP error","status":504,"url":"https://api.bandw.top/files/gia0603yucca/stage2_07070513_2datasets_construct/9cjzn0v3/file_stream","body":"error code: 504"}
+{"time":"2025-07-09T00:01:13.718163538+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": read tcp 10.1.2.136:52128->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-07-09T00:04:13.715227056+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": read tcp 10.1.2.136:56708->104.21.20.172:443: read: connection reset by peer"}
+{"time":"2025-07-09T00:41:59.079495986+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": read tcp 10.1.2.136:39968->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-07-09T00:50:28.436723591+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-07-09T01:04:28.736382048+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": read tcp 10.1.2.136:48068->104.21.20.172:443: read: connection reset by peer"}
+{"time":"2025-07-09T01:36:13.71400828+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": read tcp 10.1.2.136:33646->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-07-09T06:33:13.899246984+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-07-09T14:42:33.327607005+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07070513_2datasets_construct/9cjzn0v3/file_stream\": read tcp 10.1.2.136:36168->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-07-09T22:35:09.035751509+08:00","level":"INFO","msg":"api: retrying HTTP error","status":504,"url":"https://api.bandw.top/files/gia0603yucca/stage2_07070513_2datasets_construct/9cjzn0v3/file_stream","body":"error code: 504"}
+{"time":"2025-07-09T23:18:37.03957561+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07070513_2datasets_construct/9cjzn0v3/file_stream\": read tcp 10.1.2.136:56104->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-09T23:35:24.650683333+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07070513_2datasets_construct/9cjzn0v3/file_stream\": read tcp 10.1.2.136:56892->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-07-09T23:50:32.561736786+08:00","level":"INFO","msg":"stream: closing","id":"9cjzn0v3"}
+{"time":"2025-07-09T23:50:32.56179589+08:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2025-07-09T23:50:32.564495033+08:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2025-07-09T23:50:38.466118847+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-07-09T23:50:40.207050581+08:00","level":"INFO","msg":"handler: closed","stream_id":"9cjzn0v3"}
+{"time":"2025-07-09T23:50:40.207095276+08:00","level":"INFO","msg":"sender: closed","stream_id":"9cjzn0v3"}
+{"time":"2025-07-09T23:50:40.207092571+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"9cjzn0v3"}
+{"time":"2025-07-09T23:50:40.211547321+08:00","level":"INFO","msg":"stream: closed","id":"9cjzn0v3"}

all_checkpoints/stage2_07070513_2datasets_construct/wandb/debug.log

@@ -0,0 +1,24 @@
+2025-07-07 05:32:22,527 INFO    MainThread:9598 [wandb_setup.py:_flush():70] Current SDK version is 0.19.11
+2025-07-07 05:32:22,527 INFO    MainThread:9598 [wandb_setup.py:_flush():70] Configure stats pid to 9598
+2025-07-07 05:32:22,527 INFO    MainThread:9598 [wandb_setup.py:_flush():70] Loading settings from /root/.config/wandb/settings
+2025-07-07 05:32:22,527 INFO    MainThread:9598 [wandb_setup.py:_flush():70] Loading settings from /nas/shared/kilab/wangyujia/ProtT3/wandb/settings
+2025-07-07 05:32:22,527 INFO    MainThread:9598 [wandb_setup.py:_flush():70] Loading settings from environment variables
+2025-07-07 05:32:22,528 INFO    MainThread:9598 [wandb_init.py:setup_run_log_directory():724] Logging user logs to ./all_checkpoints/stage2_07070513_2datasets_construct/wandb/run-20250707_053222-9cjzn0v3/logs/debug.log
+2025-07-07 05:32:22,528 INFO    MainThread:9598 [wandb_init.py:setup_run_log_directory():725] Logging internal logs to ./all_checkpoints/stage2_07070513_2datasets_construct/wandb/run-20250707_053222-9cjzn0v3/logs/debug-internal.log
+2025-07-07 05:32:22,528 INFO    MainThread:9598 [wandb_init.py:init():852] calling init triggers
+2025-07-07 05:32:22,528 INFO    MainThread:9598 [wandb_init.py:init():857] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-07-07 05:32:22,528 INFO    MainThread:9598 [wandb_init.py:init():893] starting backend
+2025-07-07 05:32:22,528 INFO    MainThread:9598 [wandb_init.py:init():897] sending inform_init request
+2025-07-07 05:32:22,529 INFO    MainThread:9598 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-07-07 05:32:22,531 INFO    MainThread:9598 [wandb_init.py:init():907] backend started and connected
+2025-07-07 05:32:22,535 INFO    MainThread:9598 [wandb_init.py:init():1005] updated telemetry
+2025-07-07 05:32:22,540 INFO    MainThread:9598 [wandb_init.py:init():1029] communicating run to backend with 90.0 second timeout
+2025-07-07 05:32:24,807 INFO    MainThread:9598 [wandb_init.py:init():1104] starting run threads in backend
+2025-07-07 05:32:24,956 INFO    MainThread:9598 [wandb_run.py:_console_start():2573] atexit reg
+2025-07-07 05:32:24,956 INFO    MainThread:9598 [wandb_run.py:_redirect():2421] redirect: wrap_raw
+2025-07-07 05:32:24,959 INFO    MainThread:9598 [wandb_run.py:_redirect():2490] Wrapping output streams.
+2025-07-07 05:32:24,959 INFO    MainThread:9598 [wandb_run.py:_redirect():2513] Redirects installed.
+2025-07-07 05:32:24,961 INFO    MainThread:9598 [wandb_init.py:init():1150] run started, returning control to user process
+2025-07-07 05:32:33,644 INFO    MainThread:9598 [wandb_run.py:_config_callback():1436] config_cb None None {'filename': 'stage2_07070513_2datasets_construct', 'seed': 42, 'mode': 'train', 'strategy': 'deepspeed', 'accelerator': 'gpu', 'devices': '0,1,2,3,4,5,6,7', 'precision': 'bf16-mixed', 'max_epochs': 10, 'accumulate_grad_batches': 1, 'check_val_every_n_epoch': 1, 'enable_flash': False, 'use_wandb_logger': True, 'mix_dataset': True, 'save_every_n_epochs': 1, 'bert_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft', 'cross_attention_freq': 2, 'num_query_token': 8, 'llm_name': '/oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged', 'num_beams': 5, 'do_sample': False, 'max_inference_len': 128, 'min_inference_len': 1, 'llm_tune': 'mid_lora', 'peft_config': '', 'peft_dir': '', 'plm_model': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m', 'plm_tune': 'freeze', 'lora_r': 8, 'lora_alpha': 16, 'lora_dropout': 0.1, 'enbale_gradient_checkpointing': False, 'weight_decay': 0.05, 'init_lr': 0.0001, 'min_lr': 1e-05, 'warmup_lr': 1e-06, 'warmup_steps': 1000, 'lr_decay_rate': 0.9, 'scheduler': 'linear_warmup_cosine_lr', 'stage1_path': '/nas/shared/kilab/wangyujia/ProtT3/all_checkpoints/stage1_07041727_2dataset/epoch=29.ckpt/converted.ckpt', 'stage2_path': '', 'init_checkpoint': '', 'caption_eval_epoch': 10, 'num_workers': 8, 'batch_size': 4, 'inference_batch_size': 4, 'root': 'data', 'text_max_len': 1024, 'q_max_len': 29, 'a_max_len': 36, 'prot_max_len': 1024, 'prompt': 'The protein has the following properties: ', 'filter_side_qa': False}
+2025-07-09 23:50:32,550 INFO    MsgRouterThr:9598 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles.

all_checkpoints/stage2_07070513_2datasets_construct/wandb/run-20250707_052104-615z4bme/files/config.yaml

@@ -0,0 +1,168 @@
+_wandb:
+    value:
+        cli_version: 0.19.11
+        m:
+            - "1": trainer/global_step
+              "6":
+                - 3
+              "7": []
+            - "1": loss
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": lr
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": epoch
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+        python_version: 3.10.0
+        t:
+            "1":
+                - 1
+                - 5
+                - 9
+                - 11
+                - 33
+                - 41
+                - 49
+                - 53
+                - 55
+                - 63
+                - 103
+            "2":
+                - 1
+                - 5
+                - 9
+                - 11
+                - 33
+                - 41
+                - 49
+                - 53
+                - 55
+                - 63
+                - 103
+            "3":
+                - 7
+                - 23
+                - 55
+                - 66
+            "4": 3.10.0
+            "5": 0.19.11
+            "6": 4.52.3
+            "8":
+                - 5
+            "12": 0.19.11
+            "13": linux-x86_64
+a_max_len:
+    value: 36
+accelerator:
+    value: gpu
+accumulate_grad_batches:
+    value: 1
+batch_size:
+    value: 8
+bert_name:
+    value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft
+caption_eval_epoch:
+    value: 10
+check_val_every_n_epoch:
+    value: 1
+cross_attention_freq:
+    value: 2
+devices:
+    value: 0,1,2,3,4,5,6,7
+do_sample:
+    value: false
+enable_flash:
+    value: false
+enbale_gradient_checkpointing:
+    value: false
+filename:
+    value: stage2_07070513_2datasets_construct
+filter_side_qa:
+    value: false
+inference_batch_size:
+    value: 4
+init_checkpoint:
+    value: ""
+init_lr:
+    value: 0.0001
+llm_name:
+    value: /oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged
+llm_tune:
+    value: mid_lora
+lora_alpha:
+    value: 16
+lora_dropout:
+    value: 0.1
+lora_r:
+    value: 8
+lr_decay_rate:
+    value: 0.9
+max_epochs:
+    value: 10
+max_inference_len:
+    value: 128
+min_inference_len:
+    value: 1
+min_lr:
+    value: 1e-05
+mix_dataset:
+    value: true
+mode:
+    value: train
+num_beams:
+    value: 5
+num_query_token:
+    value: 8
+num_workers:
+    value: 8
+peft_config:
+    value: ""
+peft_dir:
+    value: ""
+plm_model:
+    value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m
+plm_tune:
+    value: freeze
+precision:
+    value: bf16-mixed
+prompt:
+    value: 'The protein has the following properties: '
+prot_max_len:
+    value: 1024
+q_max_len:
+    value: 29
+root:
+    value: data
+save_every_n_epochs:
+    value: 1
+scheduler:
+    value: linear_warmup_cosine_lr
+seed:
+    value: 42
+stage1_path:
+    value: /nas/shared/kilab/wangyujia/ProtT3/all_checkpoints/stage1_07041727_2dataset/epoch=29.ckpt/converted.ckpt
+stage2_path:
+    value: ""
+strategy:
+    value: deepspeed
+text_max_len:
+    value: 1024
+use_wandb_logger:
+    value: true
+warmup_lr:
+    value: 1e-06
+warmup_steps:
+    value: 1000
+weight_decay:
+    value: 0.05

all_checkpoints/stage2_07070513_2datasets_construct/wandb/run-20250707_052104-615z4bme/files/output.log

@@ -0,0 +1,229 @@
+/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:654: Checkpoint directory /nas/shared/kilab/wangyujia/ProtT3/all_checkpoints/stage2_07070513_2datasets_construct exists and is not empty.
+Enabling DeepSpeed BF16. Model parameters and inputs will be cast to `bfloat16`.
+LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]
+
+  | Name  | Type     | Params | Mode
+-------------------------------------------
+0 | blip2 | Blip2OPT | 7.9 B  | train
+-------------------------------------------
+104 M     Trainable params
+7.8 B     Non-trainable params
+7.9 B     Total params
+31,459.025Total estimated model params size (MB)
+174       Modules in train mode
+1203      Modules in eval mode
+Epoch 0:   1%|▌                                                | 138/13326 [03:46<6:00:39,  0.61it/s, v_num=4bme]
+
+Detected KeyboardInterrupt, attempting graceful shutdown ...
+Traceback (most recent call last):
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 47, in _call_and_handle_interrupt
+    return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 105, in launch
+    return function(*args, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 599, in _fit_impl
+    self._run(model, ckpt_path=ckpt_path)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1012, in _run
+    results = self._run_stage()
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1056, in _run_stage
+    self.fit_loop.run()
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/loops/fit_loop.py", line 216, in run
+    self.advance()
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/loops/fit_loop.py", line 455, in advance
+    self.epoch_loop.run(self._data_fetcher)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 150, in run
+    self.advance(data_fetcher)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 320, in advance
+    batch_output = self.automatic_optimization.run(trainer.optimizers[0], batch_idx, kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 192, in run
+    self._optimizer_step(batch_idx, closure)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 270, in _optimizer_step
+    call._call_lightning_module_hook(
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 176, in _call_lightning_module_hook
+    output = fn(*args, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/core/module.py", line 1302, in optimizer_step
+    optimizer.step(closure=optimizer_closure)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/core/optimizer.py", line 154, in step
+    step_output = self._strategy.optimizer_step(self._optimizer, closure, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/strategies/ddp.py", line 270, in optimizer_step
+    optimizer_output = super().optimizer_step(optimizer, closure, model, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 239, in optimizer_step
+    return self.precision_plugin.optimizer_step(optimizer, model=model, closure=closure, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/plugins/precision/deepspeed.py", line 129, in optimizer_step
+    closure_result = closure()
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 146, in __call__
+    self._result = self.closure(*args, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context
+    return func(*args, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 131, in closure
+    step_output = self._step_fn()
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 319, in _training_step
+    training_step_output = call._call_strategy_hook(trainer, "training_step", *kwargs.values())
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 328, in _call_strategy_hook
+    output = fn(*args, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 390, in training_step
+    return self._forward_redirection(self.model, self.lightning_module, "training_step", *args, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 641, in __call__
+    wrapper_output = wrapper_module(*args, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/deepspeed/utils/nvtx.py", line 20, in wrapped_fn
+    ret_val = func(*args, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/deepspeed/runtime/engine.py", line 2054, in forward
+    loss = self.module(*inputs, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1845, in _call_impl
+    return inner()
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1793, in inner
+    result = forward_call(*args, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 634, in wrapped_forward
+    out = method(*_args, **_kwargs)
+  File "/nas/shared/kilab/wangyujia/ProtT3/model/blip2_stage2.py", line 284, in training_step
+    self.log("loss", float(loss), batch_size=batch_size, sync_dist=True)
+KeyboardInterrupt
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/nas/shared/kilab/wangyujia/ProtT3/stage2.py", line 131, in <module>
+    main(get_args())
+  File "/nas/shared/kilab/wangyujia/ProtT3/stage2.py", line 93, in main
+    trainer.fit(model, datamodule=dm)#, ckpt_path=args.ckpt_path)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 561, in fit
+    call._call_and_handle_interrupt(
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 61, in _call_and_handle_interrupt
+    trainer._teardown()
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1035, in _teardown
+    self.strategy.teardown()
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/strategies/ddp.py", line 419, in teardown
+    super().teardown()
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/strategies/parallel.py", line 134, in teardown
+    super().teardown()
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 536, in teardown
+    self.lightning_module.cpu()
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/lightning_fabric/utilities/device_dtype_mixin.py", line 82, in cpu
+    return super().cpu()
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1121, in cpu
+    return self._apply(lambda t: t.cpu())
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 903, in _apply
+    module._apply(fn)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 903, in _apply
+    module._apply(fn)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 903, in _apply
+    module._apply(fn)
+  [Previous line repeated 4 more times]
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 930, in _apply
+    param_applied = fn(param)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1121, in <lambda>
+    return self._apply(lambda t: t.cpu())
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/utils/data/_utils/signal_handling.py", line 73, in handler
+    _error_if_any_worker_fails()
+RuntimeError: DataLoader worker (pid 8028) exited unexpectedly with exit code 1. Details are lost due to multiprocessing. Rerunning with num_workers=0 may give better error trace.
+[rank0]: Traceback (most recent call last):
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 47, in _call_and_handle_interrupt
+[rank0]:     return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 105, in launch
+[rank0]:     return function(*args, **kwargs)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 599, in _fit_impl
+[rank0]:     self._run(model, ckpt_path=ckpt_path)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1012, in _run
+[rank0]:     results = self._run_stage()
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1056, in _run_stage
+[rank0]:     self.fit_loop.run()
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/loops/fit_loop.py", line 216, in run
+[rank0]:     self.advance()
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/loops/fit_loop.py", line 455, in advance
+[rank0]:     self.epoch_loop.run(self._data_fetcher)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 150, in run
+[rank0]:     self.advance(data_fetcher)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 320, in advance
+[rank0]:     batch_output = self.automatic_optimization.run(trainer.optimizers[0], batch_idx, kwargs)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 192, in run
+[rank0]:     self._optimizer_step(batch_idx, closure)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 270, in _optimizer_step
+[rank0]:     call._call_lightning_module_hook(
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 176, in _call_lightning_module_hook
+[rank0]:     output = fn(*args, **kwargs)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/core/module.py", line 1302, in optimizer_step
+[rank0]:     optimizer.step(closure=optimizer_closure)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/core/optimizer.py", line 154, in step
+[rank0]:     step_output = self._strategy.optimizer_step(self._optimizer, closure, **kwargs)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/strategies/ddp.py", line 270, in optimizer_step
+[rank0]:     optimizer_output = super().optimizer_step(optimizer, closure, model, **kwargs)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 239, in optimizer_step
+[rank0]:     return self.precision_plugin.optimizer_step(optimizer, model=model, closure=closure, **kwargs)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/plugins/precision/deepspeed.py", line 129, in optimizer_step
+[rank0]:     closure_result = closure()
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 146, in __call__
+[rank0]:     self._result = self.closure(*args, **kwargs)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context
+[rank0]:     return func(*args, **kwargs)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 131, in closure
+[rank0]:     step_output = self._step_fn()
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 319, in _training_step
+[rank0]:     training_step_output = call._call_strategy_hook(trainer, "training_step", *kwargs.values())
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 328, in _call_strategy_hook
+[rank0]:     output = fn(*args, **kwargs)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 390, in training_step
+[rank0]:     return self._forward_redirection(self.model, self.lightning_module, "training_step", *args, **kwargs)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 641, in __call__
+[rank0]:     wrapper_output = wrapper_module(*args, **kwargs)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
+[rank0]:     return self._call_impl(*args, **kwargs)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
+[rank0]:     return forward_call(*args, **kwargs)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/deepspeed/utils/nvtx.py", line 20, in wrapped_fn
+[rank0]:     ret_val = func(*args, **kwargs)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/deepspeed/runtime/engine.py", line 2054, in forward
+[rank0]:     loss = self.module(*inputs, **kwargs)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
+[rank0]:     return self._call_impl(*args, **kwargs)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1845, in _call_impl
+[rank0]:     return inner()
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1793, in inner
+[rank0]:     result = forward_call(*args, **kwargs)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 634, in wrapped_forward
+[rank0]:     out = method(*_args, **_kwargs)
+[rank0]:   File "/nas/shared/kilab/wangyujia/ProtT3/model/blip2_stage2.py", line 284, in training_step
+[rank0]:     self.log("loss", float(loss), batch_size=batch_size, sync_dist=True)
+[rank0]: KeyboardInterrupt
+
+[rank0]: During handling of the above exception, another exception occurred:
+
+[rank0]: Traceback (most recent call last):
+[rank0]:   File "/nas/shared/kilab/wangyujia/ProtT3/stage2.py", line 131, in <module>
+[rank0]:     main(get_args())
+[rank0]:   File "/nas/shared/kilab/wangyujia/ProtT3/stage2.py", line 93, in main
+[rank0]:     trainer.fit(model, datamodule=dm)#, ckpt_path=args.ckpt_path)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 561, in fit
+[rank0]:     call._call_and_handle_interrupt(
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 61, in _call_and_handle_interrupt
+[rank0]:     trainer._teardown()
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1035, in _teardown
+[rank0]:     self.strategy.teardown()
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/strategies/ddp.py", line 419, in teardown
+[rank0]:     super().teardown()
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/strategies/parallel.py", line 134, in teardown
+[rank0]:     super().teardown()
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 536, in teardown
+[rank0]:     self.lightning_module.cpu()
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/lightning_fabric/utilities/device_dtype_mixin.py", line 82, in cpu
+[rank0]:     return super().cpu()
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1121, in cpu
+[rank0]:     return self._apply(lambda t: t.cpu())
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 903, in _apply
+[rank0]:     module._apply(fn)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 903, in _apply
+[rank0]:     module._apply(fn)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 903, in _apply
+[rank0]:     module._apply(fn)
+[rank0]:   [Previous line repeated 4 more times]
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 930, in _apply
+[rank0]:     param_applied = fn(param)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1121, in <lambda>
+[rank0]:     return self._apply(lambda t: t.cpu())
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/utils/data/_utils/signal_handling.py", line 73, in handler
+[rank0]:     _error_if_any_worker_fails()
+[rank0]: RuntimeError: DataLoader worker (pid 8028) exited unexpectedly with exit code 1. Details are lost due to multiprocessing. Rerunning with num_workers=0 may give better error trace.

all_checkpoints/stage2_07070513_2datasets_construct/wandb/run-20250707_052104-615z4bme/files/requirements.txt

@@ -0,0 +1,225 @@
+gitdb==4.0.12
+smmap==5.0.2
+wcwidth==0.2.13
+streamlit==1.45.1
+antlr4-python3-runtime==4.9.3
+MarkupSafe==3.0.2
+markdown-it-py==3.0.0
+PyYAML==6.0.2
+nvidia-cusolver-cu12==11.6.1.9
+text-unidecode==1.3
+msgpack==1.1.0
+pillow==11.2.1
+wrapt==1.17.2
+tifffile==2025.5.10
+nvidia-curand-cu12==10.3.5.147
+networkx==3.4.2
+fonttools==4.58.0
+plotly==6.1.1
+matplotlib==3.10.3
+certifi==2025.4.26
+altair==5.5.0
+nvidia-cufft-cu12==11.2.1.3
+ninja==1.11.1.4
+tzdata==2025.2
+nvidia-cublas-cu12==12.4.5.8
+weasel==0.4.1
+nvidia-cuda-nvrtc-cu12==12.4.127
+jedi==0.19.2
+GitPython==3.1.44
+pandas==2.2.3
+python-slugify==8.0.4
+omegaconf==2.3.0
+kiwisolver==1.4.8
+tenacity==9.1.2
+pydantic==2.11.5
+async-timeout==5.0.1
+tqdm==4.67.1
+confection==0.1.5
+six==1.17.0
+portalocker==3.1.1
+regex==2024.11.6
+nvidia-cuda-runtime-cu12==12.4.127
+packaging==24.2
+annotated-types==0.7.0
+salesforce-lavis==1.0.2
+nvidia-nvjitlink-cu12==12.4.127
+lightning-utilities==0.14.3
+pytz==2025.2
+smart-open==7.1.0
+cachetools==5.5.2
+nltk==3.9.1
+torchmetrics==1.7.1
+pexpect==4.9.0
+jsonschema-specifications==2025.4.1
+Jinja2==3.1.6
+hjson==3.1.0
+nvidia-cusparse-cu12==12.3.1.170
+nvidia-cudnn-cu12==9.1.0.70
+decord==0.6.0
+joblib==1.5.1
+kaggle==1.7.4.5
+psutil==7.0.0
+absl-py==2.2.2
+Pygments==2.19.1
+idna==3.10
+aiohappyeyeballs==2.6.1
+tornado==6.5.1
+cycler==0.12.1
+deepspeed==0.16.10+b666844f
+torchvision==0.21.0
+exceptiongroup==1.3.0
+cfgv==3.4.0
+py-cpuinfo==9.0.0
+webdataset==0.2.111
+murmurhash==1.0.13
+asttokens==3.0.0
+spacy==3.8.7
+blinker==1.9.0
+python-dateutil==2.9.0.post0
+prompt_toolkit==3.0.51
+referencing==0.36.2
+contourpy==1.3.2
+mpmath==1.3.0
+thinc==8.3.6
+pycocotools==2.0.8
+python-magic==0.4.27
+fairscale==0.4.4
+nodeenv==1.9.1
+identify==2.6.12
+ftfy==6.3.1
+spacy-legacy==3.0.12
+cymem==2.0.11
+typing-inspection==0.4.1
+nvidia-cufile-cu12==1.11.1.6
+filelock==3.18.0
+language_data==1.3.0
+iopath==0.1.10
+pre_commit==4.2.0
+toml==0.10.2
+lazy_loader==0.4
+nvidia-cusparselt-cu12==0.6.2
+nvidia-nvtx-cu12==12.4.127
+rouge_score==0.1.2
+pycocoevalcap==1.2
+pyparsing==3.2.3
+mdurl==0.1.2
+pure_eval==0.2.3
+ipython==8.36.0
+langcodes==3.5.0
+distlib==0.3.9
+pydeck==0.9.1
+traitlets==5.14.3
+decorator==5.2.1
+requests==2.32.3
+pydantic_core==2.33.2
+matplotlib-inline==0.1.7
+hf-xet==1.1.2
+opendatasets==0.1.22
+attrs==25.3.0
+urllib3==2.4.0
+typing_extensions==4.13.2
+bleach==6.2.0
+rich==14.0.0
+imageio==2.37.0
+yarl==1.20.0
+platformdirs==4.3.8
+multidict==6.4.4
+catalogue==2.0.10
+wasabi==1.1.3
+scikit-image==0.25.2
+blis==1.3.0
+pyarrow==20.0.0
+parso==0.8.4
+rpds-py==0.25.1
+opencv-python-headless==4.5.5.64
+braceexpand==0.1.7
+frozenlist==1.6.0
+numpy==2.2.6
+cloudpathlib==0.21.1
+srsly==2.5.1
+webencodings==0.5.1
+nvidia-cuda-cupti-cu12==12.4.127
+einops==0.8.1
+setuptools==78.1.1
+nvidia-nccl-cu12==2.21.5
+ptyprocess==0.7.0
+torch==2.6.0
+scipy==1.15.3
+nvidia-ml-py==12.575.51
+aiosignal==1.3.2
+virtualenv==20.31.2
+protobuf==6.31.0
+contexttimer==0.3.3
+marisa-trie==1.2.1
+shellingham==1.5.4
+charset-normalizer==3.4.2
+propcache==0.3.1
+executing==2.2.0
+pytorch-lightning==2.5.1.post0
+stack-data==0.6.3
+sentencepiece==0.2.0
+sympy==1.13.1
+wheel==0.45.1
+safetensors==0.5.3
+triton==3.2.0
+watchdog==6.0.0
+spacy-loggers==1.0.5
+timm==0.4.12
+docker-pycreds==0.4.0
+setproctitle==1.3.6
+jmespath==0.10.0
+pycryptodome==3.23.0
+opendelta==0.3.2
+aliyun-python-sdk-core==2.16.0
+dill==0.3.8
+xxhash==3.5.0
+crcmod==1.7
+aiohttp==3.12.2
+sentry-sdk==2.29.1
+huggingface-hub==0.32.1
+jaraco.functools==4.1.0
+pathlib==1.0.1
+multiprocess==0.70.16
+flash-attn==2.7.1.post1
+jsonschema==4.24.0
+datasets==3.6.0
+cffi==1.17.1
+pycparser==2.22
+fsspec==2025.3.0
+more-itertools==10.7.0
+cryptography==45.0.3
+tokenizers==0.21.1
+cheroot==10.0.1
+pip==25.1.1
+preshed==3.0.10
+transformers==4.52.3
+oss2==2.15.0
+yacs==0.1.8
+wandb==0.19.11
+bigmodelvis==0.0.1
+web.py==0.62
+opencv-python==4.11.0.86
+threadpoolctl==3.6.0
+typer==0.16.0
+narwhals==1.41.0
+delta-center-client==0.0.4
+aliyun-python-sdk-kms==2.16.5
+click==8.2.1
+scikit-learn==1.6.1
+jaraco.text==3.12.1
+autocommand==2.2.2
+packaging==24.2
+jaraco.context==5.3.0
+tomli==2.0.1
+typeguard==4.3.0
+zipp==3.19.2
+backports.tarfile==1.2.0
+typing_extensions==4.12.2
+jaraco.collections==5.1.0
+inflect==7.3.1
+more-itertools==10.3.0
+jaraco.functools==4.0.1
+importlib_metadata==8.0.0
+platformdirs==4.2.2
+wheel==0.45.1

all_checkpoints/stage2_07070513_2datasets_construct/wandb/run-20250707_052104-615z4bme/files/wandb-metadata.json

@@ -0,0 +1,104 @@
+{
+  "os": "Linux-5.10.134-008.16.kangaroo.al8.x86_64-x86_64-with-glibc2.35",
+  "python": "CPython 3.10.0",
+  "startedAt": "2025-07-06T21:21:04.133606Z",
+  "args": [
+    "--devices",
+    "0,1,2,3,4,5,6,7",
+    "--mode",
+    "train",
+    "--filename",
+    "stage2_07070513_2datasets_construct",
+    "--num_query_token",
+    "8",
+    "--save_every_n_epochs",
+    "1",
+    "--max_epochs",
+    "10",
+    "--batch_size",
+    "8",
+    "--precision",
+    "bf16-mixed",
+    "--num_workers",
+    "8",
+    "--plm_model",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m",
+    "--bert_name",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft",
+    "--llm_name",
+    "/oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged",
+    "--llm_tune",
+    "mid_lora",
+    "--mix_dataset",
+    "--stage1_path",
+    "/nas/shared/kilab/wangyujia/ProtT3/all_checkpoints/stage1_07041727_2dataset/epoch=29.ckpt/converted.ckpt",
+    "--use_wandb_logger"
+  ],
+  "program": "/nas/shared/kilab/wangyujia/ProtT3/stage2.py",
+  "codePath": "stage2.py",
+  "email": "gia0603yucca@gmail.com",
+  "root": "./all_checkpoints/stage2_07070513_2datasets_construct/",
+  "host": "dsw-251511-69b5b47496-4bcxh",
+  "executable": "/root/miniconda3/envs/protT3/bin/python",
+  "codePathLocal": "stage2.py",
+  "cpu_count": 64,
+  "cpu_count_logical": 64,
+  "gpu": "NVIDIA A800-SXM4-80GB",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1623302262784",
+      "used": "1260380160"
+    }
+  },
+  "memory": {
+    "total": "549755813888"
+  },
+  "cpu": {
+    "count": 64,
+    "countLogical": 64
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    }
+  ],
+  "cudaVersion": "12.1"
+}

all_checkpoints/stage2_07070513_2datasets_construct/wandb/run-20250707_052104-615z4bme/files/wandb-summary.json

@@ -0,0 +1 @@
+{"epoch":0,"trainer/global_step":99,"_timestamp":1.7518373296110501e+09,"_runtime":465.477642119,"_step":1,"loss":1.32979416847229,"_wandb":{"runtime":533},"lr":1.0800999916682485e-05}

all_checkpoints/stage2_07070513_2datasets_construct/wandb/run-20250707_052104-615z4bme/logs/debug-internal.log

@@ -0,0 +1,10 @@
+{"time":"2025-07-07T05:21:04.137926867+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.11","symlink path":"all_checkpoints/stage2_07070513_2datasets_construct/wandb/run-20250707_052104-615z4bme/logs/debug-core.log"}
+{"time":"2025-07-07T05:21:05.224571234+08:00","level":"INFO","msg":"created new stream","id":"615z4bme"}
+{"time":"2025-07-07T05:21:05.224615496+08:00","level":"INFO","msg":"stream: started","id":"615z4bme"}
+{"time":"2025-07-07T05:21:05.224662595+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"615z4bme"}
+{"time":"2025-07-07T05:21:05.224708291+08:00","level":"INFO","msg":"handler: started","stream_id":"615z4bme"}
+{"time":"2025-07-07T05:21:05.22467591+08:00","level":"INFO","msg":"sender: started","stream_id":"615z4bme"}
+{"time":"2025-07-07T05:21:06.409908065+08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2025-07-07T05:29:57.639658414+08:00","level":"INFO","msg":"stream: closing","id":"615z4bme"}
+{"time":"2025-07-07T05:29:57.639718652+08:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2025-07-07T05:29:57.64116529+08:00","level":"INFO","msg":"Stopped system monitor"}

all_checkpoints/stage2_07070513_2datasets_construct/wandb/run-20250707_052104-615z4bme/logs/debug.log

@@ -0,0 +1,24 @@
+2025-07-07 05:21:04,126 INFO    MainThread:2481 [wandb_setup.py:_flush():70] Current SDK version is 0.19.11
+2025-07-07 05:21:04,126 INFO    MainThread:2481 [wandb_setup.py:_flush():70] Configure stats pid to 2481
+2025-07-07 05:21:04,126 INFO    MainThread:2481 [wandb_setup.py:_flush():70] Loading settings from /root/.config/wandb/settings
+2025-07-07 05:21:04,126 INFO    MainThread:2481 [wandb_setup.py:_flush():70] Loading settings from /nas/shared/kilab/wangyujia/ProtT3/wandb/settings
+2025-07-07 05:21:04,126 INFO    MainThread:2481 [wandb_setup.py:_flush():70] Loading settings from environment variables
+2025-07-07 05:21:04,126 INFO    MainThread:2481 [wandb_init.py:setup_run_log_directory():724] Logging user logs to ./all_checkpoints/stage2_07070513_2datasets_construct/wandb/run-20250707_052104-615z4bme/logs/debug.log
+2025-07-07 05:21:04,126 INFO    MainThread:2481 [wandb_init.py:setup_run_log_directory():725] Logging internal logs to ./all_checkpoints/stage2_07070513_2datasets_construct/wandb/run-20250707_052104-615z4bme/logs/debug-internal.log
+2025-07-07 05:21:04,126 INFO    MainThread:2481 [wandb_init.py:init():852] calling init triggers
+2025-07-07 05:21:04,126 INFO    MainThread:2481 [wandb_init.py:init():857] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-07-07 05:21:04,126 INFO    MainThread:2481 [wandb_init.py:init():893] starting backend
+2025-07-07 05:21:04,126 INFO    MainThread:2481 [wandb_init.py:init():897] sending inform_init request
+2025-07-07 05:21:04,129 INFO    MainThread:2481 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-07-07 05:21:04,132 INFO    MainThread:2481 [wandb_init.py:init():907] backend started and connected
+2025-07-07 05:21:04,135 INFO    MainThread:2481 [wandb_init.py:init():1005] updated telemetry
+2025-07-07 05:21:04,138 INFO    MainThread:2481 [wandb_init.py:init():1029] communicating run to backend with 90.0 second timeout
+2025-07-07 05:21:06,399 INFO    MainThread:2481 [wandb_init.py:init():1104] starting run threads in backend
+2025-07-07 05:21:06,613 INFO    MainThread:2481 [wandb_run.py:_console_start():2573] atexit reg
+2025-07-07 05:21:06,614 INFO    MainThread:2481 [wandb_run.py:_redirect():2421] redirect: wrap_raw
+2025-07-07 05:21:06,619 INFO    MainThread:2481 [wandb_run.py:_redirect():2490] Wrapping output streams.
+2025-07-07 05:21:06,619 INFO    MainThread:2481 [wandb_run.py:_redirect():2513] Redirects installed.
+2025-07-07 05:21:06,620 INFO    MainThread:2481 [wandb_init.py:init():1150] run started, returning control to user process
+2025-07-07 05:25:59,072 INFO    MainThread:2481 [wandb_run.py:_config_callback():1436] config_cb None None {'filename': 'stage2_07070513_2datasets_construct', 'seed': 42, 'mode': 'train', 'strategy': 'deepspeed', 'accelerator': 'gpu', 'devices': '0,1,2,3,4,5,6,7', 'precision': 'bf16-mixed', 'max_epochs': 10, 'accumulate_grad_batches': 1, 'check_val_every_n_epoch': 1, 'enable_flash': False, 'use_wandb_logger': True, 'mix_dataset': True, 'save_every_n_epochs': 1, 'bert_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft', 'cross_attention_freq': 2, 'num_query_token': 8, 'llm_name': '/oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged', 'num_beams': 5, 'do_sample': False, 'max_inference_len': 128, 'min_inference_len': 1, 'llm_tune': 'mid_lora', 'peft_config': '', 'peft_dir': '', 'plm_model': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m', 'plm_tune': 'freeze', 'lora_r': 8, 'lora_alpha': 16, 'lora_dropout': 0.1, 'enbale_gradient_checkpointing': False, 'weight_decay': 0.05, 'init_lr': 0.0001, 'min_lr': 1e-05, 'warmup_lr': 1e-06, 'warmup_steps': 1000, 'lr_decay_rate': 0.9, 'scheduler': 'linear_warmup_cosine_lr', 'stage1_path': '/nas/shared/kilab/wangyujia/ProtT3/all_checkpoints/stage1_07041727_2dataset/epoch=29.ckpt/converted.ckpt', 'stage2_path': '', 'init_checkpoint': '', 'caption_eval_epoch': 10, 'num_workers': 8, 'batch_size': 8, 'inference_batch_size': 4, 'root': 'data', 'text_max_len': 1024, 'q_max_len': 29, 'a_max_len': 36, 'prot_max_len': 1024, 'prompt': 'The protein has the following properties: ', 'filter_side_qa': False}
+2025-07-07 05:29:57,621 INFO    MsgRouterThr:2481 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles.

all_checkpoints/stage2_07070513_2datasets_construct/wandb/run-20250707_052104-615z4bme/run-615z4bme.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5d1bbe7396b4e8ab1a6d6cf3abef6965bcd254b0974ca9172975647a4cc3e5a
+size 196608

all_checkpoints/stage2_07070513_2datasets_construct/wandb/run-20250707_053222-9cjzn0v3/files/config.yaml

@@ -0,0 +1,222 @@
+_wandb:
+    value:
+        cli_version: 0.19.11
+        m:
+            - "1": trainer/global_step
+              "6":
+                - 3
+              "7": []
+            - "1": loss
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": lr
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": dataset0/rouge_1
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": dataset0/rouge_l
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": dataset0/meteor_score
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": dataset0/bleu2
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": dataset0/bleu4
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": epoch
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": dataloader2/val loss/dataloader_idx_2
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": dataloader0/val loss/dataloader_idx_0
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": dataset0/acc
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": dataset0/rouge_2
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+        python_version: 3.10.0
+        t:
+            "1":
+                - 1
+                - 5
+                - 9
+                - 11
+                - 33
+                - 41
+                - 49
+                - 53
+                - 55
+                - 63
+                - 103
+            "2":
+                - 1
+                - 5
+                - 9
+                - 11
+                - 33
+                - 41
+                - 49
+                - 53
+                - 55
+                - 63
+                - 103
+            "3":
+                - 7
+                - 23
+                - 55
+                - 66
+            "4": 3.10.0
+            "5": 0.19.11
+            "6": 4.52.3
+            "8":
+                - 5
+            "12": 0.19.11
+            "13": linux-x86_64
+a_max_len:
+    value: 36
+accelerator:
+    value: gpu
+accumulate_grad_batches:
+    value: 1
+batch_size:
+    value: 4
+bert_name:
+    value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft
+caption_eval_epoch:
+    value: 10
+check_val_every_n_epoch:
+    value: 1
+cross_attention_freq:
+    value: 2
+devices:
+    value: 0,1,2,3,4,5,6,7
+do_sample:
+    value: false
+enable_flash:
+    value: false
+enbale_gradient_checkpointing:
+    value: false
+filename:
+    value: stage2_07070513_2datasets_construct
+filter_side_qa:
+    value: false
+inference_batch_size:
+    value: 4
+init_checkpoint:
+    value: ""
+init_lr:
+    value: 0.0001
+llm_name:
+    value: /oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged
+llm_tune:
+    value: mid_lora
+lora_alpha:
+    value: 16
+lora_dropout:
+    value: 0.1
+lora_r:
+    value: 8
+lr_decay_rate:
+    value: 0.9
+max_epochs:
+    value: 10
+max_inference_len:
+    value: 128
+min_inference_len:
+    value: 1
+min_lr:
+    value: 1e-05
+mix_dataset:
+    value: true
+mode:
+    value: train
+num_beams:
+    value: 5
+num_query_token:
+    value: 8
+num_workers:
+    value: 8
+peft_config:
+    value: ""
+peft_dir:
+    value: ""
+plm_model:
+    value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m
+plm_tune:
+    value: freeze
+precision:
+    value: bf16-mixed
+prompt:
+    value: 'The protein has the following properties: '
+prot_max_len:
+    value: 1024
+q_max_len:
+    value: 29
+root:
+    value: data
+save_every_n_epochs:
+    value: 1
+scheduler:
+    value: linear_warmup_cosine_lr
+seed:
+    value: 42
+stage1_path:
+    value: /nas/shared/kilab/wangyujia/ProtT3/all_checkpoints/stage1_07041727_2dataset/epoch=29.ckpt/converted.ckpt
+stage2_path:
+    value: ""
+strategy:
+    value: deepspeed
+text_max_len:
+    value: 1024
+use_wandb_logger:
+    value: true
+warmup_lr:
+    value: 1e-06
+warmup_steps:
+    value: 1000
+weight_decay:
+    value: 0.05

all_checkpoints/stage2_07070513_2datasets_construct/wandb/run-20250707_053222-9cjzn0v3/files/output.log

@@ -0,0 +1,35 @@
+/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:654: Checkpoint directory /nas/shared/kilab/wangyujia/ProtT3/all_checkpoints/stage2_07070513_2datasets_construct exists and is not empty.
+Enabling DeepSpeed BF16. Model parameters and inputs will be cast to `bfloat16`.
+LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]
+
+  | Name  | Type     | Params | Mode
+-------------------------------------------
+0 | blip2 | Blip2OPT | 7.9 B  | train
+-------------------------------------------
+104 M     Trainable params
+7.8 B     Non-trainable params
+7.9 B     Total params
+31,459.025Total estimated model params size (MB)
+174       Modules in train mode
+1203      Modules in eval mode
+Epoch 9: 100%|████████████████████████████████████████████████████████████████| 26653/26653 [6:28:32<00:00,  1.14it/s, v_num=n0v3]BLEU-2 score: 26.653377377338177
+BLEU-4 score: 20.796051979558282████████████████████████████████████████████████████████████████| 313/313 [23:19<00:00,  0.22it/s]
+/nas/shared/kilab/wangyujia/ProtT3/model/dist_funs.py:18: FutureWarning: Positional args are being deprecated, use kwargs instead. Refer to https://pytorch.org/docs/main/generated/torch.nn.Module.html#torch.nn.Module.state_dict for details.
+  sd = self.module.state_dict(destination, prefix, keep_vars)
+20000it [01:34, 212.73it/s]
+20000it [00:35, 568.59it/s]
+/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/connectors/logger_connector/result.py:434: It is recommended to use `self.log('dataset0/acc', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices.
+Average Meteor score: 27.78854434762939
+/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/connectors/logger_connector/result.py:434: It is recommended to use `self.log('dataset0/bleu2', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices.
+ROUGE score:
+rouge1: 32.64128335687479
+rouge2: 20.424108167541117
+rougeL: 27.292150023395635
+/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/connectors/logger_connector/result.py:434: It is recommended to use `self.log('dataset0/bleu4', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices.
+/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/connectors/logger_connector/result.py:434: It is recommended to use `self.log('dataset0/rouge_1', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices.
+/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/connectors/logger_connector/result.py:434: It is recommended to use `self.log('dataset0/rouge_2', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices.
+/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/connectors/logger_connector/result.py:434: It is recommended to use `self.log('dataset0/rouge_l', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices.
+/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/connectors/logger_connector/result.py:434: It is recommended to use `self.log('dataset0/meteor_score', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices.
+Epoch 9: 100%|████████████████████████████████████████████████████████████████| 26653/26653 [7:22:40<00:00,  1.00it/s, v_num=n0v3]
+
+`Trainer.fit` stopped: `max_epochs=10` reached.

all_checkpoints/stage2_07070513_2datasets_construct/wandb/run-20250707_053222-9cjzn0v3/files/requirements.txt

@@ -0,0 +1,225 @@
+gitdb==4.0.12
+smmap==5.0.2
+wcwidth==0.2.13
+streamlit==1.45.1
+antlr4-python3-runtime==4.9.3
+MarkupSafe==3.0.2
+markdown-it-py==3.0.0
+PyYAML==6.0.2
+nvidia-cusolver-cu12==11.6.1.9
+text-unidecode==1.3
+msgpack==1.1.0
+pillow==11.2.1
+wrapt==1.17.2
+tifffile==2025.5.10
+nvidia-curand-cu12==10.3.5.147
+networkx==3.4.2
+fonttools==4.58.0
+plotly==6.1.1
+matplotlib==3.10.3
+certifi==2025.4.26
+altair==5.5.0
+nvidia-cufft-cu12==11.2.1.3
+ninja==1.11.1.4
+tzdata==2025.2
+nvidia-cublas-cu12==12.4.5.8
+weasel==0.4.1
+nvidia-cuda-nvrtc-cu12==12.4.127
+jedi==0.19.2
+GitPython==3.1.44
+pandas==2.2.3
+python-slugify==8.0.4
+omegaconf==2.3.0
+kiwisolver==1.4.8
+tenacity==9.1.2
+pydantic==2.11.5
+async-timeout==5.0.1
+tqdm==4.67.1
+confection==0.1.5
+six==1.17.0
+portalocker==3.1.1
+regex==2024.11.6
+nvidia-cuda-runtime-cu12==12.4.127
+packaging==24.2
+annotated-types==0.7.0
+salesforce-lavis==1.0.2
+nvidia-nvjitlink-cu12==12.4.127
+lightning-utilities==0.14.3
+pytz==2025.2
+smart-open==7.1.0
+cachetools==5.5.2
+nltk==3.9.1
+torchmetrics==1.7.1
+pexpect==4.9.0
+jsonschema-specifications==2025.4.1
+Jinja2==3.1.6
+hjson==3.1.0
+nvidia-cusparse-cu12==12.3.1.170
+nvidia-cudnn-cu12==9.1.0.70
+decord==0.6.0
+joblib==1.5.1
+kaggle==1.7.4.5
+psutil==7.0.0
+absl-py==2.2.2
+Pygments==2.19.1
+idna==3.10
+aiohappyeyeballs==2.6.1
+tornado==6.5.1
+cycler==0.12.1
+deepspeed==0.16.10+b666844f
+torchvision==0.21.0
+exceptiongroup==1.3.0
+cfgv==3.4.0
+py-cpuinfo==9.0.0
+webdataset==0.2.111
+murmurhash==1.0.13
+asttokens==3.0.0
+spacy==3.8.7
+blinker==1.9.0
+python-dateutil==2.9.0.post0
+prompt_toolkit==3.0.51
+referencing==0.36.2
+contourpy==1.3.2
+mpmath==1.3.0
+thinc==8.3.6
+pycocotools==2.0.8
+python-magic==0.4.27
+fairscale==0.4.4
+nodeenv==1.9.1
+identify==2.6.12
+ftfy==6.3.1
+spacy-legacy==3.0.12
+cymem==2.0.11
+typing-inspection==0.4.1
+nvidia-cufile-cu12==1.11.1.6
+filelock==3.18.0
+language_data==1.3.0
+iopath==0.1.10
+pre_commit==4.2.0
+toml==0.10.2
+lazy_loader==0.4
+nvidia-cusparselt-cu12==0.6.2
+nvidia-nvtx-cu12==12.4.127
+rouge_score==0.1.2
+pycocoevalcap==1.2
+pyparsing==3.2.3
+mdurl==0.1.2
+pure_eval==0.2.3
+ipython==8.36.0
+langcodes==3.5.0
+distlib==0.3.9
+pydeck==0.9.1
+traitlets==5.14.3
+decorator==5.2.1
+requests==2.32.3
+pydantic_core==2.33.2
+matplotlib-inline==0.1.7
+hf-xet==1.1.2
+opendatasets==0.1.22
+attrs==25.3.0
+urllib3==2.4.0
+typing_extensions==4.13.2
+bleach==6.2.0
+rich==14.0.0
+imageio==2.37.0
+yarl==1.20.0
+platformdirs==4.3.8
+multidict==6.4.4
+catalogue==2.0.10
+wasabi==1.1.3
+scikit-image==0.25.2
+blis==1.3.0
+pyarrow==20.0.0
+parso==0.8.4
+rpds-py==0.25.1
+opencv-python-headless==4.5.5.64
+braceexpand==0.1.7
+frozenlist==1.6.0
+numpy==2.2.6
+cloudpathlib==0.21.1
+srsly==2.5.1
+webencodings==0.5.1
+nvidia-cuda-cupti-cu12==12.4.127
+einops==0.8.1
+setuptools==78.1.1
+nvidia-nccl-cu12==2.21.5
+ptyprocess==0.7.0
+torch==2.6.0
+scipy==1.15.3
+nvidia-ml-py==12.575.51
+aiosignal==1.3.2
+virtualenv==20.31.2
+protobuf==6.31.0
+contexttimer==0.3.3
+marisa-trie==1.2.1
+shellingham==1.5.4
+charset-normalizer==3.4.2
+propcache==0.3.1
+executing==2.2.0
+pytorch-lightning==2.5.1.post0
+stack-data==0.6.3
+sentencepiece==0.2.0
+sympy==1.13.1
+wheel==0.45.1
+safetensors==0.5.3
+triton==3.2.0
+watchdog==6.0.0
+spacy-loggers==1.0.5
+timm==0.4.12
+docker-pycreds==0.4.0
+setproctitle==1.3.6
+jmespath==0.10.0
+pycryptodome==3.23.0
+opendelta==0.3.2
+aliyun-python-sdk-core==2.16.0
+dill==0.3.8
+xxhash==3.5.0
+crcmod==1.7
+aiohttp==3.12.2
+sentry-sdk==2.29.1
+huggingface-hub==0.32.1
+jaraco.functools==4.1.0
+pathlib==1.0.1
+multiprocess==0.70.16
+flash-attn==2.7.1.post1
+jsonschema==4.24.0
+datasets==3.6.0
+cffi==1.17.1
+pycparser==2.22
+fsspec==2025.3.0
+more-itertools==10.7.0
+cryptography==45.0.3
+tokenizers==0.21.1
+cheroot==10.0.1
+pip==25.1.1
+preshed==3.0.10
+transformers==4.52.3
+oss2==2.15.0
+yacs==0.1.8
+wandb==0.19.11
+bigmodelvis==0.0.1
+web.py==0.62
+opencv-python==4.11.0.86
+threadpoolctl==3.6.0
+typer==0.16.0
+narwhals==1.41.0
+delta-center-client==0.0.4
+aliyun-python-sdk-kms==2.16.5
+click==8.2.1
+scikit-learn==1.6.1
+jaraco.text==3.12.1
+autocommand==2.2.2
+packaging==24.2
+jaraco.context==5.3.0
+tomli==2.0.1
+typeguard==4.3.0
+zipp==3.19.2
+backports.tarfile==1.2.0
+typing_extensions==4.12.2
+jaraco.collections==5.1.0
+inflect==7.3.1
+more-itertools==10.3.0
+jaraco.functools==4.0.1
+importlib_metadata==8.0.0
+platformdirs==4.2.2
+wheel==0.45.1

all_checkpoints/stage2_07070513_2datasets_construct/wandb/run-20250707_053222-9cjzn0v3/files/wandb-metadata.json

@@ -0,0 +1,104 @@
+{
+  "os": "Linux-5.10.134-008.16.kangaroo.al8.x86_64-x86_64-with-glibc2.35",
+  "python": "CPython 3.10.0",
+  "startedAt": "2025-07-06T21:32:22.534976Z",
+  "args": [
+    "--devices",
+    "0,1,2,3,4,5,6,7",
+    "--mode",
+    "train",
+    "--filename",
+    "stage2_07070513_2datasets_construct",
+    "--num_query_token",
+    "8",
+    "--save_every_n_epochs",
+    "1",
+    "--max_epochs",
+    "10",
+    "--batch_size",
+    "4",
+    "--precision",
+    "bf16-mixed",
+    "--num_workers",
+    "8",
+    "--plm_model",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m",
+    "--bert_name",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft",
+    "--llm_name",
+    "/oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged",
+    "--llm_tune",
+    "mid_lora",
+    "--mix_dataset",
+    "--stage1_path",
+    "/nas/shared/kilab/wangyujia/ProtT3/all_checkpoints/stage1_07041727_2dataset/epoch=29.ckpt/converted.ckpt",
+    "--use_wandb_logger"
+  ],
+  "program": "/nas/shared/kilab/wangyujia/ProtT3/stage2.py",
+  "codePath": "stage2.py",
+  "email": "gia0603yucca@gmail.com",
+  "root": "./all_checkpoints/stage2_07070513_2datasets_construct/",
+  "host": "dsw-251511-69b5b47496-4bcxh",
+  "executable": "/root/miniconda3/envs/protT3/bin/python",
+  "codePathLocal": "stage2.py",
+  "cpu_count": 64,
+  "cpu_count_logical": 64,
+  "gpu": "NVIDIA A800-SXM4-80GB",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1623302262784",
+      "used": "1260400640"
+    }
+  },
+  "memory": {
+    "total": "549755813888"
+  },
+  "cpu": {
+    "count": 64,
+    "countLogical": 64
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    }
+  ],
+  "cudaVersion": "12.1"
+}

all_checkpoints/stage2_07070513_2datasets_construct/wandb/run-20250707_053222-9cjzn0v3/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_runtime":238677.754852379,"trainer/global_step":266529,"dataloader0/val loss/dataloader_idx_0":0.3741031885147095,"dataset0/acc":0,"dataset0/meteor_score":27.788543701171875,"_wandb":{"runtime":238690},"dataset0/bleu4":20.796052932739258,"dataset0/rouge_2":20.424108505249023,"lr":1.2202456673549023e-05,"_timestamp":1.7520762202895813e+09,"dataloader2/val loss/dataloader_idx_2":0.2067195624113083,"dataset0/rouge_1":32.64128494262695,"loss":0.114682637155056,"epoch":9,"dataset0/bleu2":26.653377532958984,"dataset0/rouge_l":27.292150497436523,"_step":5339}

all_checkpoints/stage2_07070513_2datasets_construct/wandb/run-20250707_053222-9cjzn0v3/logs/debug-internal.log

@@ -0,0 +1,29 @@
+{"time":"2025-07-07T05:32:22.544190733+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.11","symlink path":"all_checkpoints/stage2_07070513_2datasets_construct/wandb/run-20250707_053222-9cjzn0v3/logs/debug-core.log"}
+{"time":"2025-07-07T05:32:23.597067843+08:00","level":"INFO","msg":"created new stream","id":"9cjzn0v3"}
+{"time":"2025-07-07T05:32:23.59711309+08:00","level":"INFO","msg":"stream: started","id":"9cjzn0v3"}
+{"time":"2025-07-07T05:32:23.59715533+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"9cjzn0v3"}
+{"time":"2025-07-07T05:32:23.597176058+08:00","level":"INFO","msg":"handler: started","stream_id":"9cjzn0v3"}
+{"time":"2025-07-07T05:32:23.597249736+08:00","level":"INFO","msg":"sender: started","stream_id":"9cjzn0v3"}
+{"time":"2025-07-07T05:32:24.815832776+08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2025-07-07T16:23:26.191588391+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07070513_2datasets_construct/9cjzn0v3/file_stream\": read tcp 10.1.2.136:46082->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-08T07:36:01.662714436+08:00","level":"INFO","msg":"api: retrying HTTP error","status":504,"url":"https://api.bandw.top/files/gia0603yucca/stage2_07070513_2datasets_construct/9cjzn0v3/file_stream","body":"error code: 504"}
+{"time":"2025-07-08T07:39:35.510926561+08:00","level":"INFO","msg":"api: retrying HTTP error","status":504,"url":"https://api.bandw.top/files/gia0603yucca/stage2_07070513_2datasets_construct/9cjzn0v3/file_stream","body":"error code: 504"}
+{"time":"2025-07-09T00:01:13.718163538+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": read tcp 10.1.2.136:52128->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-07-09T00:04:13.715227056+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": read tcp 10.1.2.136:56708->104.21.20.172:443: read: connection reset by peer"}
+{"time":"2025-07-09T00:41:59.079495986+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": read tcp 10.1.2.136:39968->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-07-09T00:50:28.436723591+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-07-09T01:04:28.736382048+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": read tcp 10.1.2.136:48068->104.21.20.172:443: read: connection reset by peer"}
+{"time":"2025-07-09T01:36:13.71400828+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": read tcp 10.1.2.136:33646->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-07-09T06:33:13.899246984+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-07-09T14:42:33.327607005+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07070513_2datasets_construct/9cjzn0v3/file_stream\": read tcp 10.1.2.136:36168->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-07-09T22:35:09.035751509+08:00","level":"INFO","msg":"api: retrying HTTP error","status":504,"url":"https://api.bandw.top/files/gia0603yucca/stage2_07070513_2datasets_construct/9cjzn0v3/file_stream","body":"error code: 504"}
+{"time":"2025-07-09T23:18:37.03957561+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07070513_2datasets_construct/9cjzn0v3/file_stream\": read tcp 10.1.2.136:56104->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-09T23:35:24.650683333+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07070513_2datasets_construct/9cjzn0v3/file_stream\": read tcp 10.1.2.136:56892->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-07-09T23:50:32.561736786+08:00","level":"INFO","msg":"stream: closing","id":"9cjzn0v3"}
+{"time":"2025-07-09T23:50:32.56179589+08:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2025-07-09T23:50:32.564495033+08:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2025-07-09T23:50:38.466118847+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-07-09T23:50:40.207050581+08:00","level":"INFO","msg":"handler: closed","stream_id":"9cjzn0v3"}
+{"time":"2025-07-09T23:50:40.207095276+08:00","level":"INFO","msg":"sender: closed","stream_id":"9cjzn0v3"}
+{"time":"2025-07-09T23:50:40.207092571+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"9cjzn0v3"}
+{"time":"2025-07-09T23:50:40.211547321+08:00","level":"INFO","msg":"stream: closed","id":"9cjzn0v3"}

all_checkpoints/stage2_07070513_2datasets_construct/wandb/run-20250707_053222-9cjzn0v3/logs/debug.log

@@ -0,0 +1,24 @@
+2025-07-07 05:32:22,527 INFO    MainThread:9598 [wandb_setup.py:_flush():70] Current SDK version is 0.19.11
+2025-07-07 05:32:22,527 INFO    MainThread:9598 [wandb_setup.py:_flush():70] Configure stats pid to 9598
+2025-07-07 05:32:22,527 INFO    MainThread:9598 [wandb_setup.py:_flush():70] Loading settings from /root/.config/wandb/settings
+2025-07-07 05:32:22,527 INFO    MainThread:9598 [wandb_setup.py:_flush():70] Loading settings from /nas/shared/kilab/wangyujia/ProtT3/wandb/settings
+2025-07-07 05:32:22,527 INFO    MainThread:9598 [wandb_setup.py:_flush():70] Loading settings from environment variables
+2025-07-07 05:32:22,528 INFO    MainThread:9598 [wandb_init.py:setup_run_log_directory():724] Logging user logs to ./all_checkpoints/stage2_07070513_2datasets_construct/wandb/run-20250707_053222-9cjzn0v3/logs/debug.log
+2025-07-07 05:32:22,528 INFO    MainThread:9598 [wandb_init.py:setup_run_log_directory():725] Logging internal logs to ./all_checkpoints/stage2_07070513_2datasets_construct/wandb/run-20250707_053222-9cjzn0v3/logs/debug-internal.log
+2025-07-07 05:32:22,528 INFO    MainThread:9598 [wandb_init.py:init():852] calling init triggers
+2025-07-07 05:32:22,528 INFO    MainThread:9598 [wandb_init.py:init():857] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-07-07 05:32:22,528 INFO    MainThread:9598 [wandb_init.py:init():893] starting backend
+2025-07-07 05:32:22,528 INFO    MainThread:9598 [wandb_init.py:init():897] sending inform_init request
+2025-07-07 05:32:22,529 INFO    MainThread:9598 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-07-07 05:32:22,531 INFO    MainThread:9598 [wandb_init.py:init():907] backend started and connected
+2025-07-07 05:32:22,535 INFO    MainThread:9598 [wandb_init.py:init():1005] updated telemetry
+2025-07-07 05:32:22,540 INFO    MainThread:9598 [wandb_init.py:init():1029] communicating run to backend with 90.0 second timeout
+2025-07-07 05:32:24,807 INFO    MainThread:9598 [wandb_init.py:init():1104] starting run threads in backend
+2025-07-07 05:32:24,956 INFO    MainThread:9598 [wandb_run.py:_console_start():2573] atexit reg
+2025-07-07 05:32:24,956 INFO    MainThread:9598 [wandb_run.py:_redirect():2421] redirect: wrap_raw
+2025-07-07 05:32:24,959 INFO    MainThread:9598 [wandb_run.py:_redirect():2490] Wrapping output streams.
+2025-07-07 05:32:24,959 INFO    MainThread:9598 [wandb_run.py:_redirect():2513] Redirects installed.
+2025-07-07 05:32:24,961 INFO    MainThread:9598 [wandb_init.py:init():1150] run started, returning control to user process
+2025-07-07 05:32:33,644 INFO    MainThread:9598 [wandb_run.py:_config_callback():1436] config_cb None None {'filename': 'stage2_07070513_2datasets_construct', 'seed': 42, 'mode': 'train', 'strategy': 'deepspeed', 'accelerator': 'gpu', 'devices': '0,1,2,3,4,5,6,7', 'precision': 'bf16-mixed', 'max_epochs': 10, 'accumulate_grad_batches': 1, 'check_val_every_n_epoch': 1, 'enable_flash': False, 'use_wandb_logger': True, 'mix_dataset': True, 'save_every_n_epochs': 1, 'bert_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft', 'cross_attention_freq': 2, 'num_query_token': 8, 'llm_name': '/oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged', 'num_beams': 5, 'do_sample': False, 'max_inference_len': 128, 'min_inference_len': 1, 'llm_tune': 'mid_lora', 'peft_config': '', 'peft_dir': '', 'plm_model': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m', 'plm_tune': 'freeze', 'lora_r': 8, 'lora_alpha': 16, 'lora_dropout': 0.1, 'enbale_gradient_checkpointing': False, 'weight_decay': 0.05, 'init_lr': 0.0001, 'min_lr': 1e-05, 'warmup_lr': 1e-06, 'warmup_steps': 1000, 'lr_decay_rate': 0.9, 'scheduler': 'linear_warmup_cosine_lr', 'stage1_path': '/nas/shared/kilab/wangyujia/ProtT3/all_checkpoints/stage1_07041727_2dataset/epoch=29.ckpt/converted.ckpt', 'stage2_path': '', 'init_checkpoint': '', 'caption_eval_epoch': 10, 'num_workers': 8, 'batch_size': 4, 'inference_batch_size': 4, 'root': 'data', 'text_max_len': 1024, 'q_max_len': 29, 'a_max_len': 36, 'prot_max_len': 1024, 'prompt': 'The protein has the following properties: ', 'filter_side_qa': False}
+2025-07-09 23:50:32,550 INFO    MsgRouterThr:9598 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles.

all_checkpoints/stage2_07070513_2datasets_construct/wandb/run-20250707_053222-9cjzn0v3/run-9cjzn0v3.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d6cffb771629ea66d100de0be9a1ef1c3f9599c478045c824d755e2ea04fe379
+size 199737973

all_checkpoints/stage2_07301646_2datasets_construct/epoch=03.ckpt/checkpoint/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:246a0d8dbc7414f986a333879b13c36671f129b3b117d7b4066f3928cb35bc99
+size 156403632

all_checkpoints/stage2_07301646_2datasets_construct/epoch=03.ckpt/checkpoint/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e2c9fc89299dca46abb0b74a86e998a1f7b2026cca0ce2dcf590da1a68df2186
+size 156402992

all_checkpoints/stage2_07301646_2datasets_construct/epoch=03.ckpt/checkpoint/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b9d65b0b283d3d437c8eb4955242a571b50d7503bb4d371742052fe466db312
+size 156403376

all_checkpoints/stage2_07301646_2datasets_construct/epoch=03.ckpt/checkpoint/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:735a857a56751f34bbcf2516fd6b038ff0617777b78e86a12ba3d5d4181e8119
+size 156403120

all_checkpoints/stage2_07301646_2datasets_construct/epoch=03.ckpt/checkpoint/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e14097721553b19f34683b33ddb67fa3d670e035d7260159b9e97aac7d7e851c
+size 156402416

all_checkpoints/stage2_07301646_2datasets_construct/epoch=03.ckpt/checkpoint/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe993f0a43264f371f9feef67c3aaf51480074bf4512a075fbcf26c11b405c6d
+size 156403696

all_checkpoints/stage2_07301646_2datasets_construct/epoch=03.ckpt/checkpoint/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:094d7605822331acc11f1cf58534bc7d48c4a66bd686ba9749042f97fe8def0e
+size 156402992

all_checkpoints/stage2_07301646_2datasets_construct/epoch=03.ckpt/checkpoint/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79ebf1ed60ade77ba2453feda74fbc5d71c1d55b9e7d67f1398ea6e6e7d45041
+size 156417904

all_checkpoints/stage2_07301646_2datasets_construct/epoch=03.ckpt/checkpoint/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e5f1a07851a422c60545e9fa30da0dc892ea9ef21aa34e05d2e11791664d89b6
+size 208795320

all_checkpoints/stage2_07301646_2datasets_construct/epoch=03.ckpt/converted.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f74c2b94f722a6ce4ebc2444d8247a103295f9673fbfab2529712ca70294557
+size 417200548