Add files using upload-large-folder tool

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+all_checkpoints/stage1_06290009_deepspeed/wandb/run-20250629_001200-vgvxxzqc/run-vgvxxzqc.wandb filter=lfs diff=lfs merge=lfs -text
+all_checkpoints/stage1_07041727_2dataset/wandb/run-20250704_172854-6bkqzmou/run-6bkqzmou.wandb filter=lfs diff=lfs merge=lfs -text
+all_checkpoints/stage2_07070337_2datasets_noconstruct/wandb/run-20250707_041231-gtrtcbb9/run-gtrtcbb9.wandb filter=lfs diff=lfs merge=lfs -text

=0.12.10

@@ -0,0 +1,33 @@
+Looking in indexes: https://mirrors.aliyun.com/pypi/simple/
+Collecting wandb
+  Downloading https://mirrors.aliyun.com/pypi/packages/f9/31/eeb2878b26566c04c3e9b8b20b3ec3c54a2be50535088d36a37c008e07a3/wandb-0.19.11-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (21.4 MB)
+     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 21.4/21.4 MB 6.2 MB/s eta 0:00:00
+Requirement already satisfied: click!=8.0.0,>=7.1 in /root/miniconda3/envs/protT3/lib/python3.10/site-packages (from wandb) (8.2.1)
+Collecting docker-pycreds>=0.4.0 (from wandb)
+  Downloading https://mirrors.aliyun.com/pypi/packages/f5/e8/f6bd1eee09314e7e6dee49cbe2c5e22314ccdb38db16c9fc72d2fa80d054/docker_pycreds-0.4.0-py2.py3-none-any.whl (9.0 kB)
+Requirement already satisfied: gitpython!=3.1.29,>=1.0.0 in /root/miniconda3/envs/protT3/lib/python3.10/site-packages (from wandb) (3.1.44)
+Requirement already satisfied: platformdirs in /root/miniconda3/envs/protT3/lib/python3.10/site-packages (from wandb) (4.3.8)
+Requirement already satisfied: protobuf!=4.21.0,!=5.28.0,<7,>=3.19.0 in /root/miniconda3/envs/protT3/lib/python3.10/site-packages (from wandb) (6.31.0)
+Requirement already satisfied: psutil>=5.0.0 in /root/miniconda3/envs/protT3/lib/python3.10/site-packages (from wandb) (7.0.0)
+Requirement already satisfied: pydantic<3 in /root/miniconda3/envs/protT3/lib/python3.10/site-packages (from wandb) (2.11.5)
+Requirement already satisfied: pyyaml in /root/miniconda3/envs/protT3/lib/python3.10/site-packages (from wandb) (6.0.2)
+Requirement already satisfied: requests<3,>=2.0.0 in /root/miniconda3/envs/protT3/lib/python3.10/site-packages (from wandb) (2.32.3)
+Collecting sentry-sdk>=2.0.0 (from wandb)
+  Downloading https://mirrors.aliyun.com/pypi/packages/f0/e5/da07b0bd832cefd52d16f2b9bbbe31624d57552602c06631686b93ccb1bd/sentry_sdk-2.29.1-py2.py3-none-any.whl (341 kB)
+Collecting setproctitle (from wandb)
+  Downloading https://mirrors.aliyun.com/pypi/packages/67/2b/c3cbd4a4462c1143465d8c151f1d51bbfb418e60a96a754329d28d416575/setproctitle-1.3.6-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (30 kB)
+Requirement already satisfied: setuptools in /root/miniconda3/envs/protT3/lib/python3.10/site-packages (from wandb) (78.1.1)
+Requirement already satisfied: typing-extensions<5,>=4.4 in /root/miniconda3/envs/protT3/lib/python3.10/site-packages (from wandb) (4.13.2)
+Requirement already satisfied: annotated-types>=0.6.0 in /root/miniconda3/envs/protT3/lib/python3.10/site-packages (from pydantic<3->wandb) (0.7.0)
+Requirement already satisfied: pydantic-core==2.33.2 in /root/miniconda3/envs/protT3/lib/python3.10/site-packages (from pydantic<3->wandb) (2.33.2)
+Requirement already satisfied: typing-inspection>=0.4.0 in /root/miniconda3/envs/protT3/lib/python3.10/site-packages (from pydantic<3->wandb) (0.4.1)
+Requirement already satisfied: charset-normalizer<4,>=2 in /root/miniconda3/envs/protT3/lib/python3.10/site-packages (from requests<3,>=2.0.0->wandb) (3.4.2)
+Requirement already satisfied: idna<4,>=2.5 in /root/miniconda3/envs/protT3/lib/python3.10/site-packages (from requests<3,>=2.0.0->wandb) (3.10)
+Requirement already satisfied: urllib3<3,>=1.21.1 in /root/miniconda3/envs/protT3/lib/python3.10/site-packages (from requests<3,>=2.0.0->wandb) (2.4.0)
+Requirement already satisfied: certifi>=2017.4.17 in /root/miniconda3/envs/protT3/lib/python3.10/site-packages (from requests<3,>=2.0.0->wandb) (2025.4.26)
+Requirement already satisfied: six>=1.4.0 in /root/miniconda3/envs/protT3/lib/python3.10/site-packages (from docker-pycreds>=0.4.0->wandb) (1.17.0)
+Requirement already satisfied: gitdb<5,>=4.0.1 in /root/miniconda3/envs/protT3/lib/python3.10/site-packages (from gitpython!=3.1.29,>=1.0.0->wandb) (4.0.12)
+Requirement already satisfied: smmap<6,>=3.0.1 in /root/miniconda3/envs/protT3/lib/python3.10/site-packages (from gitdb<5,>=4.0.1->gitpython!=3.1.29,>=1.0.0->wandb) (5.0.2)
+Installing collected packages: setproctitle, sentry-sdk, docker-pycreds, wandb
+
+Successfully installed docker-pycreds-0.4.0 sentry-sdk-2.29.1 setproctitle-1.3.6 wandb-0.19.11

README.md

@@ -0,0 +1,105 @@
+# ProtT3: Protein-to-Text Generation for Text-based Protein Understanding
+
+Codes of our ACL2024 paper.
+
+Authors: Zhiyuan Liu, An Zhang, Hao Fei, Enzhi Zhang, Xiang Wang, Kenji Kawaguchi, Tat-Seng Chua
+
+
+## Dependencies
+
+python==3.8
+
+* Install PyTorch with cuda-11.7 using conda by following the instructions in [link](https://pytorch.org/get-started/locally/)
+* Install flash-attention by running `pip install flash-attn --no-build-isolation`. You might need to install the following dependencies first, for building the flash-attention module:
+    * `pip install packaging ninja`
+    * `conda install -c "nvidia/label/cuda-11.7.1" cuda-nvcc`
+    * `conda install -c "nvidia/label/cuda-11.7.1" cuda-libraries-dev`
+* Install the lastest version of opendela by runing `pip install git+https://github.com/thunlp/OpenDelta.git`
+* Install Lavis: `pip install rouge_score nltk salesforce-lavis`
+* Install others: `pip install -U transformers pytorch-lightning`
+* Install the lastest version of deepspeed: `pip install git+https://github.com/microsoft/DeepSpeed.git`
+* Download nltk corpus:
+```
+import nltk
+nltk.download('wordnet')
+```
+
+## Dataset
+
+Download our pre-processed datasets from [link](https://osf.io/23azs/?view_only=185575515e714f4798499bf06513a730), and unzip the datasets under the `./data` directory
+
+## Reproduce results by training from scratch
+
+* Reproduce results in stage 1:
+
+```sh
+python stage1.py --devices '0,1,2,3' --mode train --filename stage1_ckpt --num_query_token 8 --plm_name "facebook/esm2_t30_150M_UR50D" --save_every_n_epochs 10 --batch_size 32 --precision 'bf16-mixed' --num_workers 8
+```
+
+* Convert stage1's DeepSpeed checkpoint to PyTorch format by running
+
+```sh
+python convert.py --input /path/to/stage1/ckpt/address --output /path/to/ckpt/saving/address
+```
+
+* Reproduce results in stage 2:
+
+    * Protein Captioning:
+
+        ```sh
+        python stage2.py --devices '0,1,2,3' --mode train --filename protein_captioning_swiss_dataset --num_query_token 8  --save_every_n_epochs 10 --batch_size 32 --precision 'bf16-mixed' --num_workers 8 --llm_tune mid_lora --enable_flash --root './data/SwissProtV3' --stage1_path /path/to/ckpt/saving/address;
+        ```
+
+    * Protein Question-Answering:
+
+        ```sh
+        python stage2.py --devices '0,1,2,3' --mode train  --filename prot_qa --num_query_token 8  --save_every_n_epochs 10 --num_workers 8 --batch_size 128 --accumulate_grad_batches 1 --precision 'bf16-mixed'  --root "data/PDBDataset" --llm_tune mid_lora --prompt "Question: {} Answer:" --inference_batch 32 --max_inference_len 36  --stage1_path /path/to/ckpt/saving/address;
+        ```
+
+    * After running one of the two scripts above, the model's protein-to-text generation resuults will be saved at `./all_checkpoint/[filename]/lightning_logs/[version_x]/dataset0_predictions.txt`. You can evaluate the results by running
+
+        ```sh 
+        ## for question-answering evaluation
+        python read_results --path ./all_checkpoint/[filename]/lightning_logs/[version_x]/dataset0_predictions.txt --qa_question 
+        
+        ## for protein captioning evaluation
+        python read_results --path ./all_checkpoint/[filename]/lightning_logs/[version_x]/dataset0_predictions.txt 
+        ```
+
+## Reproduce results by loading our checkpoints
+
+Download our released checkpoints from [link](https://osf.io/23azs/?view_only=185575515e714f4798499bf06513a730)
+
+* Reproduce results in stage 1:
+
+```sh
+python stage1.py --devices '0,1,2,3' --mode eval --filename stage1_ckpt --num_query_token 8 --plm_name "facebook/esm2_t30_150M_UR50D" --save_every_n_epochs 10 --batch_size 32 --precision 'bf16-mixed' --num_workers 8 --init_checkpoint /path/to/stage1.ckpt;
+```
+
+* Reproduce results in stage 2:
+
+    * Protein Captioning:
+
+        ```sh
+        python stage2.py --devices '0,1,2,3' --mode train --filename protein_captioning_swiss_dataset --num_query_token 8  --save_every_n_epochs 10 --batch_size 32 --precision 'bf16-mixed' --num_workers 8 --llm_tune mid_lora --enable_flash --root './data/SwissProtV3' --init_checkpoint /path/to/swiss_ft.ckpt;
+        ```
+
+    * Protein Question-Answering:
+
+        ```sh
+        python stage2.py --devices '0,1,2,3' --mode train  --filename prot_qa --num_query_token 8  --save_every_n_epochs 10 --num_workers 8 --batch_size 128 --accumulate_grad_batches 1 --precision 'bf16-mixed'  --root "data/PDBDataset" --llm_tune mid_lora --prompt "Question: {} Answer:" --inference_batch 32 --max_inference_len 36  --init_checkpoint /path/to/pdbqa_ft.ckpt;
+        ```
+
+
+## Citation
+
+```bib
+@inproceedings{liu2024prott,
+    title={ProtT3: Protein-to-Text Generation for Text-based Protein Understanding},
+    author={Liu, Zhiyuan and Zhang, An and Fei, Hao and Zhang, Enzhi and Wang, Xiang and Kawaguchi, Kenji and Chua, Tat-Seng},
+    booktitle={{ACL}},
+    publisher    = {Association for Computational Linguistics},
+    year={2024},
+    url={https://openreview.net/forum?id=ZmIjOPil2b}
+}
+```

all_checkpoints/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

all_checkpoints/stage1_06290009_deepspeed/epoch=19.ckpt/checkpoint/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1d6798603170f2f7903bb9a0d21a5ed25e860528296107c8b31cd2914a86965
+size 269461360

all_checkpoints/stage1_06290009_deepspeed/epoch=19.ckpt/checkpoint/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ab0580d136844b18a0e388946d57dca52ab782052d7885d0c033a07708b3957
+size 269466288

all_checkpoints/stage1_06290009_deepspeed/epoch=19.ckpt/checkpoint/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e71ea3d389287f043d6acb2922ee075f8ceaf30ac73b145c5cb709ca3a4cfea1
+size 269466352

all_checkpoints/stage1_06290009_deepspeed/epoch=19.ckpt/checkpoint/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9bd10e7db6ec37fcbdb2baa6f51691e815bde2e5a112b516675fb556c4674e87
+size 269465648

all_checkpoints/stage1_06290009_deepspeed/epoch=19.ckpt/checkpoint/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b5975d03715bc1cc0eb9de733fbb4475c0540e790c6f00d0aae625e8ba08230
+size 269466096

all_checkpoints/stage1_06290009_deepspeed/epoch=19.ckpt/checkpoint/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61d27b7a3de97f24ef54d58dc7e2a57d2568ed520177746f47540c3f4d26a847
+size 269466352

all_checkpoints/stage1_06290009_deepspeed/epoch=19.ckpt/checkpoint/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08bcb9b3d8e2b0e8e71b925b4c0ef0da6702e9609c252dc90607d9d4e3801b4c
+size 269466096

all_checkpoints/stage1_06290009_deepspeed/epoch=19.ckpt/checkpoint/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d426f595bbbde8b5a76ead76e06cee6febe35c7cf6edcf40aaf06560d47100e
+size 269465904

all_checkpoints/stage1_06290009_deepspeed/epoch=19.ckpt/checkpoint/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2691cb5de812f5198012ce9c319ee4faae20c95a97a5639b3796d13561d44b49
+size 359448824

all_checkpoints/stage1_06290009_deepspeed/epoch=19.ckpt/converted.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aabff8ac8cffa262ba0de81d44b32342f1fce2f1b6efe66064bb9d99b3ef01bd
+size 718651620

all_checkpoints/stage1_06290009_deepspeed/epoch=19.ckpt/latest

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47320987f9a49d5b00119b960f247a956773f57543982b8bfcb6da5bb3afd9ef
+size 10

all_checkpoints/stage1_06290009_deepspeed/epoch=19.ckpt/zero_to_fp32.py

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:46497565ccf2b4a8b1f6f18c8341042f3749605a94335c81f69df1bd268af64f
+size 33272

all_checkpoints/stage1_06290009_deepspeed/wandb/debug-internal.log

@@ -0,0 +1,42 @@
+{"time":"2025-06-29T00:12:00.930911849+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.11","symlink path":"all_checkpoints/stage1_06290009_deepspeed/wandb/run-20250629_001200-vgvxxzqc/logs/debug-core.log"}
+{"time":"2025-06-29T00:12:31.03707486+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:12:36.876219526+08:00","level":"INFO","msg":"created new stream","id":"vgvxxzqc"}
+{"time":"2025-06-29T00:12:36.876272436+08:00","level":"INFO","msg":"stream: started","id":"vgvxxzqc"}
+{"time":"2025-06-29T00:12:36.876317878+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"vgvxxzqc"}
+{"time":"2025-06-29T00:12:36.876360145+08:00","level":"INFO","msg":"handler: started","stream_id":"vgvxxzqc"}
+{"time":"2025-06-29T00:12:36.876401621+08:00","level":"INFO","msg":"sender: started","stream_id":"vgvxxzqc"}
+{"time":"2025-06-29T00:12:39.839397838+08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2025-06-29T00:13:20.87652364+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:58014->104.21.20.172:443: read: connection reset by peer"}
+{"time":"2025-06-29T00:16:17.426969211+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": unexpected EOF"}
+{"time":"2025-06-29T00:17:58.51721346+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": unexpected EOF"}
+{"time":"2025-06-29T00:26:10.222503445+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:51478->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-06-29T00:28:34.807681677+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:46066->104.21.20.172:443: read: connection reset by peer"}
+{"time":"2025-06-29T00:33:01.870533298+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:55734->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-29T00:33:23.754842161+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:46042->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-06-29T00:36:47.149592254+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:57176->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-06-29T00:41:05.645712211+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": unexpected EOF"}
+{"time":"2025-06-29T00:47:20.493525378+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:58300->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-29T00:47:45.629981285+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": unexpected EOF"}
+{"time":"2025-06-29T00:51:19.597480154+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:60802->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-29T00:54:09.777365701+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:51532->104.21.20.172:443: read: connection reset by peer"}
+{"time":"2025-06-29T01:00:20.78154967+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:34934->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-06-29T01:04:21.421531776+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:55938->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-29T01:05:41.05509194+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:45678->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-06-29T01:08:44.130266043+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": unexpected EOF"}
+{"time":"2025-06-29T01:10:59.724692621+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": unexpected EOF"}
+{"time":"2025-06-29T01:14:35.821570745+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:35232->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-29T01:17:35.533530754+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:44258->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-29T01:23:12.630381225+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:48108->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-06-29T01:24:55.067569821+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T01:25:27.188065501+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T01:25:56.782489942+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:45024->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-29T01:26:49.538892815+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": unexpected EOF"}
+{"time":"2025-06-29T01:29:46.157546022+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:42508->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-06-29T05:53:57.9412544+08:00","level":"INFO","msg":"stream: closing","id":"vgvxxzqc"}
+{"time":"2025-06-29T05:53:57.941286983+08:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2025-06-29T05:53:57.942366437+08:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2025-06-29T05:54:00.869660002+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-06-29T05:54:03.731237694+08:00","level":"INFO","msg":"handler: closed","stream_id":"vgvxxzqc"}
+{"time":"2025-06-29T05:54:03.731282348+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"vgvxxzqc"}
+{"time":"2025-06-29T05:54:03.731313818+08:00","level":"INFO","msg":"sender: closed","stream_id":"vgvxxzqc"}
+{"time":"2025-06-29T05:54:03.735031072+08:00","level":"INFO","msg":"stream: closed","id":"vgvxxzqc"}

all_checkpoints/stage1_06290009_deepspeed/wandb/debug.log

@@ -0,0 +1,24 @@
+2025-06-29 00:12:00,920 INFO    MainThread:2351 [wandb_setup.py:_flush():70] Current SDK version is 0.19.11
+2025-06-29 00:12:00,920 INFO    MainThread:2351 [wandb_setup.py:_flush():70] Configure stats pid to 2351
+2025-06-29 00:12:00,920 INFO    MainThread:2351 [wandb_setup.py:_flush():70] Loading settings from /root/.config/wandb/settings
+2025-06-29 00:12:00,920 INFO    MainThread:2351 [wandb_setup.py:_flush():70] Loading settings from /nas/shared/kilab/wangyujia/ProtT3/wandb/settings
+2025-06-29 00:12:00,920 INFO    MainThread:2351 [wandb_setup.py:_flush():70] Loading settings from environment variables
+2025-06-29 00:12:00,920 INFO    MainThread:2351 [wandb_init.py:setup_run_log_directory():724] Logging user logs to ./all_checkpoints/stage1_06290009_deepspeed/wandb/run-20250629_001200-vgvxxzqc/logs/debug.log
+2025-06-29 00:12:00,920 INFO    MainThread:2351 [wandb_init.py:setup_run_log_directory():725] Logging internal logs to ./all_checkpoints/stage1_06290009_deepspeed/wandb/run-20250629_001200-vgvxxzqc/logs/debug-internal.log
+2025-06-29 00:12:00,920 INFO    MainThread:2351 [wandb_init.py:init():852] calling init triggers
+2025-06-29 00:12:00,921 INFO    MainThread:2351 [wandb_init.py:init():857] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-06-29 00:12:00,921 INFO    MainThread:2351 [wandb_init.py:init():893] starting backend
+2025-06-29 00:12:00,921 INFO    MainThread:2351 [wandb_init.py:init():897] sending inform_init request
+2025-06-29 00:12:00,923 INFO    MainThread:2351 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-06-29 00:12:00,924 INFO    MainThread:2351 [wandb_init.py:init():907] backend started and connected
+2025-06-29 00:12:00,926 INFO    MainThread:2351 [wandb_init.py:init():1005] updated telemetry
+2025-06-29 00:12:00,929 INFO    MainThread:2351 [wandb_init.py:init():1029] communicating run to backend with 90.0 second timeout
+2025-06-29 00:12:39,788 INFO    MainThread:2351 [wandb_init.py:init():1104] starting run threads in backend
+2025-06-29 00:12:40,030 INFO    MainThread:2351 [wandb_run.py:_console_start():2573] atexit reg
+2025-06-29 00:12:40,030 INFO    MainThread:2351 [wandb_run.py:_redirect():2421] redirect: wrap_raw
+2025-06-29 00:12:40,034 INFO    MainThread:2351 [wandb_run.py:_redirect():2490] Wrapping output streams.
+2025-06-29 00:12:40,034 INFO    MainThread:2351 [wandb_run.py:_redirect():2513] Redirects installed.
+2025-06-29 00:12:40,036 INFO    MainThread:2351 [wandb_init.py:init():1150] run started, returning control to user process
+2025-06-29 00:12:46,669 INFO    MainThread:2351 [wandb_run.py:_config_callback():1436] config_cb None None {'filename': 'stage1_06290009_deepspeed', 'seed': 42, 'mode': 'train', 'strategy': 'deepspeed', 'accelerator': 'gpu', 'devices': '0,1,2,3,4,5,6,7', 'precision': 'bf16-mixed', 'max_epochs': 20, 'check_val_every_n_epoch': 1, 'use_wandb_logger': True, 'mix_dataset': True, 'temperature': 0.1, 'save_every_n_epochs': 5, 'ptm': True, 'lm': True, 'rerank_cand_num': 128, 'plm_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m', 'plm_tune': 'freeze', 'load_4bit': False, 'pool_size': 0, 'bert_hidden_dim': 768, 'bert_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft', 'projection_dim': 256, 'cross_attention_freq': 2, 'num_query_token': 8, 'weight_decay': 0.05, 'init_lr': 0.0001, 'min_lr': 1e-05, 'warmup_lr': 1e-06, 'warmup_steps': 1000, 'lr_decay_rate': 0.9, 'scheduler': 'linear_warmup_cosine_lr', 'init_checkpoint': '', 'retrieval_eval_epoch': 10, 'num_workers': 8, 'batch_size': 168, 'match_batch_size': 64, 'root': 'data', 'text_max_len': 128, 'prot_max_len': 1024, 'prot_aug': 'None'}
+2025-06-29 05:53:57,929 INFO    MsgRouterThr:2351 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 2 handles.

all_checkpoints/stage1_06290009_deepspeed/wandb/run-20250629_001200-vgvxxzqc/files/config.yaml

@@ -0,0 +1,236 @@
+_wandb:
+    value:
+        cli_version: 0.19.11
+        m:
+            - "1": trainer/global_step
+              "6":
+                - 3
+              "7": []
+            - "1": train_loss
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loader1/val_loss/dataloader_idx_1
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loader0/val_loss_lm/dataloader_idx_0
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loader1/val_loss_ptm/dataloader_idx_1
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loader2/val_loss_lm/dataloader_idx_2
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loader1/val_loss_ptc/dataloader_idx_1
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": epoch
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": lr
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loader2/val_loss_ptc/dataloader_idx_2
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loader0/val_loss_ptm/dataloader_idx_0
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": train_loss_ptc
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": train_loss_ptm
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": train_loss_lm
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loader2/val_loss_ptm/dataloader_idx_2
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loader0/val_loss/dataloader_idx_0
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loader2/val_loss/dataloader_idx_2
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loader1/val_loss_lm/dataloader_idx_1
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loader0/val_loss_ptc/dataloader_idx_0
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+        python_version: 3.10.0
+        t:
+            "1":
+                - 1
+                - 5
+                - 9
+                - 11
+                - 33
+                - 41
+                - 49
+                - 53
+                - 55
+                - 63
+                - 103
+            "2":
+                - 1
+                - 5
+                - 9
+                - 11
+                - 33
+                - 41
+                - 49
+                - 53
+                - 55
+                - 63
+                - 103
+            "3":
+                - 7
+                - 23
+                - 55
+                - 66
+            "4": 3.10.0
+            "5": 0.19.11
+            "6": 4.52.3
+            "8":
+                - 5
+            "12": 0.19.11
+            "13": linux-x86_64
+accelerator:
+    value: gpu
+batch_size:
+    value: 168
+bert_hidden_dim:
+    value: 768
+bert_name:
+    value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft
+check_val_every_n_epoch:
+    value: 1
+cross_attention_freq:
+    value: 2
+devices:
+    value: 0,1,2,3,4,5,6,7
+filename:
+    value: stage1_06290009_deepspeed
+init_checkpoint:
+    value: ""
+init_lr:
+    value: 0.0001
+lm:
+    value: true
+load_4bit:
+    value: false
+lr_decay_rate:
+    value: 0.9
+match_batch_size:
+    value: 64
+max_epochs:
+    value: 20
+min_lr:
+    value: 1e-05
+mix_dataset:
+    value: true
+mode:
+    value: train
+num_query_token:
+    value: 8
+num_workers:
+    value: 8
+plm_name:
+    value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m
+plm_tune:
+    value: freeze
+pool_size:
+    value: 0
+precision:
+    value: bf16-mixed
+projection_dim:
+    value: 256
+prot_aug:
+    value: None
+prot_max_len:
+    value: 1024
+ptm:
+    value: true
+rerank_cand_num:
+    value: 128
+retrieval_eval_epoch:
+    value: 10
+root:
+    value: data
+save_every_n_epochs:
+    value: 5
+scheduler:
+    value: linear_warmup_cosine_lr
+seed:
+    value: 42
+strategy:
+    value: deepspeed
+temperature:
+    value: 0.1
+text_max_len:
+    value: 128
+use_wandb_logger:
+    value: true
+warmup_lr:
+    value: 1e-06
+warmup_steps:
+    value: 1000
+weight_decay:
+    value: 0.05

all_checkpoints/stage1_06290009_deepspeed/wandb/run-20250629_001200-vgvxxzqc/files/output.log

@@ -0,0 +1,21 @@
+/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:654: Checkpoint directory /nas/shared/kilab/wangyujia/ProtT3/all_checkpoints/stage1_06290009_deepspeed exists and is not empty.
+Enabling DeepSpeed BF16. Model parameters and inputs will be cast to `bfloat16`.
+LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]
+
+  | Name         | Type         | Params | Mode
+------------------------------------------------------
+0 | blip2qformer | Blip2Qformer | 327 M  | train
+------------------------------------------------------
+179 M     Trainable params
+147 M     Non-trainable params
+327 M     Total params
+1,309.467 Total estimated model params size (MB)
+5         Modules in train mode
+926       Modules in eval mode
+Epoch 19: 100%|███████████████████████████████████████████| 320/320 [17:07<00:00,  0.31it/s, v_num=xzqc]
+/nas/shared/kilab/wangyujia/ProtT3/model/blip2qformer.py:220: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(enable_autocast, dtype=torch.float32):
+                                                                                                        
+/nas/shared/kilab/wangyujia/ProtT3/model/dist_funs.py:18: FutureWarning: Positional args are being deprecated, use kwargs instead. Refer to https://pytorch.org/docs/main/generated/torch.nn.Module.html#torch.nn.Module.state_dict for details.
+  sd = self.module.state_dict(destination, prefix, keep_vars)
+`Trainer.fit` stopped: `max_epochs=20` reached.

all_checkpoints/stage1_06290009_deepspeed/wandb/run-20250629_001200-vgvxxzqc/files/requirements.txt

@@ -0,0 +1,225 @@
+opendatasets==0.1.22
+salesforce-lavis==1.0.2
+Pygments==2.19.1
+nvidia-nccl-cu12==2.21.5
+tornado==6.5.1
+nvidia-cuda-runtime-cu12==12.4.127
+requests==2.32.3
+nvidia-cuda-cupti-cu12==12.4.127
+decord==0.6.0
+braceexpand==0.1.7
+frozenlist==1.6.0
+markdown-it-py==3.0.0
+shellingham==1.5.4
+absl-py==2.2.2
+pycocoevalcap==1.2
+contexttimer==0.3.3
+bleach==6.2.0
+jsonschema-specifications==2025.4.1
+pycocotools==2.0.8
+python-slugify==8.0.4
+tqdm==4.67.1
+numpy==2.2.6
+urllib3==2.4.0
+deepspeed==0.16.10+b666844f
+watchdog==6.0.0
+wrapt==1.17.2
+setuptools==78.1.1
+matplotlib==3.10.3
+pydeck==0.9.1
+aiosignal==1.3.2
+gitdb==4.0.12
+hjson==3.1.0
+timm==0.4.12
+blis==1.3.0
+PyYAML==6.0.2
+referencing==0.36.2
+contourpy==1.3.2
+kaggle==1.7.4.5
+triton==3.2.0
+catalogue==2.0.10
+idna==3.10
+torch==2.6.0
+text-unidecode==1.3
+altair==5.5.0
+cloudpathlib==0.21.1
+protobuf==6.31.0
+nvidia-cusolver-cu12==11.6.1.9
+pytz==2025.2
+sympy==1.13.1
+spacy==3.8.7
+MarkupSafe==3.0.2
+thinc==8.3.6
+nvidia-cudnn-cu12==9.1.0.70
+wasabi==1.1.3
+aiohappyeyeballs==2.6.1
+nvidia-nvtx-cu12==12.4.127
+rich==14.0.0
+ipython==8.36.0
+yarl==1.20.0
+torchmetrics==1.7.1
+multidict==6.4.4
+cfgv==3.4.0
+smmap==5.0.2
+srsly==2.5.1
+scikit-image==0.25.2
+matplotlib-inline==0.1.7
+annotated-types==0.7.0
+lazy_loader==0.4
+tenacity==9.1.2
+GitPython==3.1.44
+language_data==1.3.0
+pydantic_core==2.33.2
+sentencepiece==0.2.0
+platformdirs==4.3.8
+distlib==0.3.9
+nvidia-cusparselt-cu12==0.6.2
+blinker==1.9.0
+regex==2024.11.6
+tifffile==2025.5.10
+py-cpuinfo==9.0.0
+attrs==25.3.0
+mdurl==0.1.2
+prompt_toolkit==3.0.51
+packaging==24.2
+async-timeout==5.0.1
+six==1.17.0
+executing==2.2.0
+parso==0.8.4
+omegaconf==2.3.0
+wcwidth==0.2.13
+murmurhash==1.0.13
+stack-data==0.6.3
+nvidia-cufft-cu12==11.2.1.3
+virtualenv==20.31.2
+langcodes==3.5.0
+fonttools==4.58.0
+opencv-python-headless==4.5.5.64
+jedi==0.19.2
+torchvision==0.21.0
+plotly==6.1.1
+nodeenv==1.9.1
+smart-open==7.1.0
+toml==0.10.2
+pytorch-lightning==2.5.1.post0
+typing_extensions==4.13.2
+safetensors==0.5.3
+psutil==7.0.0
+pillow==11.2.1
+python-dateutil==2.9.0.post0
+ftfy==6.3.1
+scipy==1.15.3
+webdataset==0.2.111
+charset-normalizer==3.4.2
+nvidia-nvjitlink-cu12==12.4.127
+kiwisolver==1.4.8
+nvidia-ml-py==12.575.51
+confection==0.1.5
+nvidia-curand-cu12==10.3.5.147
+pandas==2.2.3
+nltk==3.9.1
+webencodings==0.5.1
+pyarrow==20.0.0
+asttokens==3.0.0
+exceptiongroup==1.3.0
+pre_commit==4.2.0
+ninja==1.11.1.4
+spacy-loggers==1.0.5
+msgpack==1.1.0
+lightning-utilities==0.14.3
+nvidia-cublas-cu12==12.4.5.8
+tzdata==2025.2
+cycler==0.12.1
+hf-xet==1.1.2
+antlr4-python3-runtime==4.9.3
+iopath==0.1.10
+pexpect==4.9.0
+imageio==2.37.0
+streamlit==1.45.1
+python-magic==0.4.27
+networkx==3.4.2
+portalocker==3.1.1
+nvidia-cusparse-cu12==12.3.1.170
+propcache==0.3.1
+ptyprocess==0.7.0
+fairscale==0.4.4
+rpds-py==0.25.1
+certifi==2025.4.26
+rouge_score==0.1.2
+traitlets==5.14.3
+identify==2.6.12
+spacy-legacy==3.0.12
+weasel==0.4.1
+mpmath==1.3.0
+cymem==2.0.11
+typing-inspection==0.4.1
+nvidia-cuda-nvrtc-cu12==12.4.127
+marisa-trie==1.2.1
+einops==0.8.1
+nvidia-cufile-cu12==1.11.1.6
+pydantic==2.11.5
+cachetools==5.5.2
+joblib==1.5.1
+Jinja2==3.1.6
+filelock==3.18.0
+pyparsing==3.2.3
+pure_eval==0.2.3
+decorator==5.2.1
+wheel==0.45.1
+pycryptodome==3.23.0
+cheroot==10.0.1
+multiprocess==0.70.16
+aiohttp==3.12.2
+crcmod==1.7
+fsspec==2025.3.0
+jmespath==0.10.0
+preshed==3.0.10
+jaraco.functools==4.1.0
+cryptography==45.0.3
+sentry-sdk==2.29.1
+tokenizers==0.21.1
+opendelta==0.3.2
+pycparser==2.22
+narwhals==1.41.0
+scikit-learn==1.6.1
+dill==0.3.8
+oss2==2.15.0
+yacs==0.1.8
+more-itertools==10.7.0
+pip==25.1.1
+threadpoolctl==3.6.0
+flash-attn==2.7.1.post1
+bigmodelvis==0.0.1
+pathlib==1.0.1
+delta-center-client==0.0.4
+xxhash==3.5.0
+wandb==0.19.11
+setproctitle==1.3.6
+aliyun-python-sdk-core==2.16.0
+transformers==4.52.3
+aliyun-python-sdk-kms==2.16.5
+datasets==3.6.0
+typer==0.16.0
+docker-pycreds==0.4.0
+click==8.2.1
+huggingface-hub==0.32.1
+web.py==0.62
+cffi==1.17.1
+opencv-python==4.11.0.86
+jsonschema==4.24.0
+typing_extensions==4.12.2
+jaraco.functools==4.0.1
+jaraco.text==3.12.1
+jaraco.collections==5.1.0
+inflect==7.3.1
+more-itertools==10.3.0
+packaging==24.2
+importlib_metadata==8.0.0
+backports.tarfile==1.2.0
+typeguard==4.3.0
+zipp==3.19.2
+platformdirs==4.2.2
+autocommand==2.2.2
+jaraco.context==5.3.0
+tomli==2.0.1
+wheel==0.45.1

all_checkpoints/stage1_06290009_deepspeed/wandb/run-20250629_001200-vgvxxzqc/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os": "Linux-5.10.134-008.16.kangaroo.al8.x86_64-x86_64-with-glibc2.35",
+  "python": "CPython 3.10.0",
+  "startedAt": "2025-06-28T16:12:00.926076Z",
+  "args": [
+    "--devices",
+    "0,1,2,3,4,5,6,7",
+    "--mode",
+    "train",
+    "--filename",
+    "stage1_06290009_deepspeed",
+    "--num_query_token",
+    "8",
+    "--plm_name",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m",
+    "--bert_name",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft",
+    "--save_every_n_epochs",
+    "5",
+    "--max_epochs",
+    "20",
+    "--batch_size",
+    "168",
+    "--precision",
+    "bf16-mixed",
+    "--mix_dataset",
+    "--num_workers",
+    "8",
+    "--use_wandb_logger"
+  ],
+  "program": "/nas/shared/kilab/wangyujia/ProtT3/stage1.py",
+  "codePath": "stage1.py",
+  "root": "./all_checkpoints/stage1_06290009_deepspeed/",
+  "host": "dsw-251511-c65bb988c-9g24f",
+  "executable": "/root/miniconda3/envs/protT3/bin/python",
+  "codePathLocal": "stage1.py",
+  "cpu_count": 64,
+  "cpu_count_logical": 64,
+  "gpu": "NVIDIA A800-SXM4-80GB",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1623302262784",
+      "used": "987680768"
+    }
+  },
+  "memory": {
+    "total": "549755813888"
+  },
+  "cpu": {
+    "count": 64,
+    "countLogical": 64
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    }
+  ],
+  "cudaVersion": "12.1"
+}

all_checkpoints/stage1_06290009_deepspeed/wandb/run-20250629_001200-vgvxxzqc/files/wandb-summary.json

@@ -0,0 +1 @@
+{"loader1/val_loss_lm/dataloader_idx_1":5.460818767547607,"lr":1.0554024811426643e-05,"_timestamp":1.7511476307831557e+09,"loader1/val_loss/dataloader_idx_1":14.773687362670898,"train_loss":2.630859375,"loader2/val_loss_ptm/dataloader_idx_2":2.988206148147583,"loader1/val_loss_ptm/dataloader_idx_1":3.491912603378296,"loader2/val_loss_lm/dataloader_idx_2":6.094737529754639,"_runtime":20509.857471191,"loader2/val_loss_ptc/dataloader_idx_2":5.59119987487793,"_wandb":{"runtime":20517},"train_loss_ptc":0.64306640625,"train_loss_ptm":0.291015625,"loader0/val_loss_ptc/dataloader_idx_0":1.0085703134536743,"loader1/val_loss_ptc/dataloader_idx_1":5.825062274932861,"loader0/val_loss_lm/dataloader_idx_0":2.1440062522888184,"loader2/val_loss/dataloader_idx_2":14.674175262451172,"epoch":19,"train_loss_lm":1.697265625,"_step":147,"loader0/val_loss/dataloader_idx_0":3.6699562072753906,"loader0/val_loss_ptm/dataloader_idx_0":0.5172882676124573,"trainer/global_step":6399}

all_checkpoints/stage1_06290009_deepspeed/wandb/run-20250629_001200-vgvxxzqc/logs/debug-internal.log

@@ -0,0 +1,42 @@
+{"time":"2025-06-29T00:12:00.930911849+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.11","symlink path":"all_checkpoints/stage1_06290009_deepspeed/wandb/run-20250629_001200-vgvxxzqc/logs/debug-core.log"}
+{"time":"2025-06-29T00:12:31.03707486+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:12:36.876219526+08:00","level":"INFO","msg":"created new stream","id":"vgvxxzqc"}
+{"time":"2025-06-29T00:12:36.876272436+08:00","level":"INFO","msg":"stream: started","id":"vgvxxzqc"}
+{"time":"2025-06-29T00:12:36.876317878+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"vgvxxzqc"}
+{"time":"2025-06-29T00:12:36.876360145+08:00","level":"INFO","msg":"handler: started","stream_id":"vgvxxzqc"}
+{"time":"2025-06-29T00:12:36.876401621+08:00","level":"INFO","msg":"sender: started","stream_id":"vgvxxzqc"}
+{"time":"2025-06-29T00:12:39.839397838+08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2025-06-29T00:13:20.87652364+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:58014->104.21.20.172:443: read: connection reset by peer"}
+{"time":"2025-06-29T00:16:17.426969211+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": unexpected EOF"}
+{"time":"2025-06-29T00:17:58.51721346+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": unexpected EOF"}
+{"time":"2025-06-29T00:26:10.222503445+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:51478->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-06-29T00:28:34.807681677+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:46066->104.21.20.172:443: read: connection reset by peer"}
+{"time":"2025-06-29T00:33:01.870533298+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:55734->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-29T00:33:23.754842161+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:46042->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-06-29T00:36:47.149592254+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:57176->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-06-29T00:41:05.645712211+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": unexpected EOF"}
+{"time":"2025-06-29T00:47:20.493525378+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:58300->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-29T00:47:45.629981285+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": unexpected EOF"}
+{"time":"2025-06-29T00:51:19.597480154+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:60802->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-29T00:54:09.777365701+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:51532->104.21.20.172:443: read: connection reset by peer"}
+{"time":"2025-06-29T01:00:20.78154967+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:34934->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-06-29T01:04:21.421531776+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:55938->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-29T01:05:41.05509194+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:45678->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-06-29T01:08:44.130266043+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": unexpected EOF"}
+{"time":"2025-06-29T01:10:59.724692621+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": unexpected EOF"}
+{"time":"2025-06-29T01:14:35.821570745+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:35232->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-29T01:17:35.533530754+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:44258->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-29T01:23:12.630381225+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:48108->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-06-29T01:24:55.067569821+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T01:25:27.188065501+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T01:25:56.782489942+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:45024->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-29T01:26:49.538892815+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": unexpected EOF"}
+{"time":"2025-06-29T01:29:46.157546022+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:42508->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-06-29T05:53:57.9412544+08:00","level":"INFO","msg":"stream: closing","id":"vgvxxzqc"}
+{"time":"2025-06-29T05:53:57.941286983+08:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2025-06-29T05:53:57.942366437+08:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2025-06-29T05:54:00.869660002+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-06-29T05:54:03.731237694+08:00","level":"INFO","msg":"handler: closed","stream_id":"vgvxxzqc"}
+{"time":"2025-06-29T05:54:03.731282348+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"vgvxxzqc"}
+{"time":"2025-06-29T05:54:03.731313818+08:00","level":"INFO","msg":"sender: closed","stream_id":"vgvxxzqc"}
+{"time":"2025-06-29T05:54:03.735031072+08:00","level":"INFO","msg":"stream: closed","id":"vgvxxzqc"}

all_checkpoints/stage1_06290009_deepspeed/wandb/run-20250629_001200-vgvxxzqc/logs/debug.log

@@ -0,0 +1,24 @@
+2025-06-29 00:12:00,920 INFO    MainThread:2351 [wandb_setup.py:_flush():70] Current SDK version is 0.19.11
+2025-06-29 00:12:00,920 INFO    MainThread:2351 [wandb_setup.py:_flush():70] Configure stats pid to 2351
+2025-06-29 00:12:00,920 INFO    MainThread:2351 [wandb_setup.py:_flush():70] Loading settings from /root/.config/wandb/settings
+2025-06-29 00:12:00,920 INFO    MainThread:2351 [wandb_setup.py:_flush():70] Loading settings from /nas/shared/kilab/wangyujia/ProtT3/wandb/settings
+2025-06-29 00:12:00,920 INFO    MainThread:2351 [wandb_setup.py:_flush():70] Loading settings from environment variables
+2025-06-29 00:12:00,920 INFO    MainThread:2351 [wandb_init.py:setup_run_log_directory():724] Logging user logs to ./all_checkpoints/stage1_06290009_deepspeed/wandb/run-20250629_001200-vgvxxzqc/logs/debug.log
+2025-06-29 00:12:00,920 INFO    MainThread:2351 [wandb_init.py:setup_run_log_directory():725] Logging internal logs to ./all_checkpoints/stage1_06290009_deepspeed/wandb/run-20250629_001200-vgvxxzqc/logs/debug-internal.log
+2025-06-29 00:12:00,920 INFO    MainThread:2351 [wandb_init.py:init():852] calling init triggers
+2025-06-29 00:12:00,921 INFO    MainThread:2351 [wandb_init.py:init():857] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-06-29 00:12:00,921 INFO    MainThread:2351 [wandb_init.py:init():893] starting backend
+2025-06-29 00:12:00,921 INFO    MainThread:2351 [wandb_init.py:init():897] sending inform_init request
+2025-06-29 00:12:00,923 INFO    MainThread:2351 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-06-29 00:12:00,924 INFO    MainThread:2351 [wandb_init.py:init():907] backend started and connected
+2025-06-29 00:12:00,926 INFO    MainThread:2351 [wandb_init.py:init():1005] updated telemetry
+2025-06-29 00:12:00,929 INFO    MainThread:2351 [wandb_init.py:init():1029] communicating run to backend with 90.0 second timeout
+2025-06-29 00:12:39,788 INFO    MainThread:2351 [wandb_init.py:init():1104] starting run threads in backend
+2025-06-29 00:12:40,030 INFO    MainThread:2351 [wandb_run.py:_console_start():2573] atexit reg
+2025-06-29 00:12:40,030 INFO    MainThread:2351 [wandb_run.py:_redirect():2421] redirect: wrap_raw
+2025-06-29 00:12:40,034 INFO    MainThread:2351 [wandb_run.py:_redirect():2490] Wrapping output streams.
+2025-06-29 00:12:40,034 INFO    MainThread:2351 [wandb_run.py:_redirect():2513] Redirects installed.
+2025-06-29 00:12:40,036 INFO    MainThread:2351 [wandb_init.py:init():1150] run started, returning control to user process
+2025-06-29 00:12:46,669 INFO    MainThread:2351 [wandb_run.py:_config_callback():1436] config_cb None None {'filename': 'stage1_06290009_deepspeed', 'seed': 42, 'mode': 'train', 'strategy': 'deepspeed', 'accelerator': 'gpu', 'devices': '0,1,2,3,4,5,6,7', 'precision': 'bf16-mixed', 'max_epochs': 20, 'check_val_every_n_epoch': 1, 'use_wandb_logger': True, 'mix_dataset': True, 'temperature': 0.1, 'save_every_n_epochs': 5, 'ptm': True, 'lm': True, 'rerank_cand_num': 128, 'plm_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m', 'plm_tune': 'freeze', 'load_4bit': False, 'pool_size': 0, 'bert_hidden_dim': 768, 'bert_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft', 'projection_dim': 256, 'cross_attention_freq': 2, 'num_query_token': 8, 'weight_decay': 0.05, 'init_lr': 0.0001, 'min_lr': 1e-05, 'warmup_lr': 1e-06, 'warmup_steps': 1000, 'lr_decay_rate': 0.9, 'scheduler': 'linear_warmup_cosine_lr', 'init_checkpoint': '', 'retrieval_eval_epoch': 10, 'num_workers': 8, 'batch_size': 168, 'match_batch_size': 64, 'root': 'data', 'text_max_len': 128, 'prot_max_len': 1024, 'prot_aug': 'None'}
+2025-06-29 05:53:57,929 INFO    MsgRouterThr:2351 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 2 handles.

all_checkpoints/stage1_06290009_deepspeed/wandb/run-20250629_001200-vgvxxzqc/run-vgvxxzqc.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02b9bae29f7eff7d1456d46d7a5fc028ecfa83d57215ed5ead79c22a39602cb8
+size 7831005

all_checkpoints/stage1_07041727_2dataset/epoch=19.ckpt/checkpoint/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eef5f0c622ec387ea2170b4d959435d6318f6de5a9cc1feb0516c58c1a678732
+size 269461360

all_checkpoints/stage1_07041727_2dataset/epoch=19.ckpt/checkpoint/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db4bea839400c6bc8cdb4aeecb922ce12df2e2d61dd5044829ecbda842413c2a
+size 269466288

all_checkpoints/stage1_07041727_2dataset/epoch=19.ckpt/checkpoint/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:15d343b92ad55cb228b7571bad35ee7aaebdbedbcfe4160f83250484719333b3
+size 269466352

all_checkpoints/stage1_07041727_2dataset/epoch=19.ckpt/checkpoint/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d1f3393bc0e7b8baae8bec392cd9d8599815f083eb9c2bc478332a65f1aff728
+size 269465648

all_checkpoints/stage1_07041727_2dataset/epoch=19.ckpt/checkpoint/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a24b3d97ae95d888cfdd074705605bf04315162a2c87780e94c7d6fedbca9b49
+size 269466096

all_checkpoints/stage1_07041727_2dataset/epoch=19.ckpt/checkpoint/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:208d3f152953084a010524ab98240fdece461a004a016925d05ac6e3f92847a6
+size 269466352

all_checkpoints/stage1_07041727_2dataset/epoch=19.ckpt/checkpoint/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8860a1be45ca900d2da827852856116e0a548ce82432d94210d4713cbbd5acc4
+size 269466096

all_checkpoints/stage1_07041727_2dataset/epoch=19.ckpt/checkpoint/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d33924194e0954a8e36ef63297fdc2b9754a2e9f368e8f302cab40552b8d5f20
+size 269465904

all_checkpoints/stage1_07041727_2dataset/epoch=19.ckpt/checkpoint/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b7ded329a604392d5b75d24097eff999add88eee739da2b384cb7b6a8b5c311a
+size 359448824

all_checkpoints/stage1_07041727_2dataset/epoch=19.ckpt/latest

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47320987f9a49d5b00119b960f247a956773f57543982b8bfcb6da5bb3afd9ef
+size 10

all_checkpoints/stage1_07041727_2dataset/epoch=19.ckpt/zero_to_fp32.py

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:46497565ccf2b4a8b1f6f18c8341042f3749605a94335c81f69df1bd268af64f
+size 33272

all_checkpoints/stage1_07041727_2dataset/epoch=29.ckpt/checkpoint/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:194d691a6ab90298daf6d5eaa20e3f9a8164e46912823a9fef49bf8b38aad782
+size 269461360

all_checkpoints/stage1_07041727_2dataset/epoch=29.ckpt/checkpoint/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c4f598e2f0e66dbdc116fd5ca5e5311fe8b80a8f685e59a9a09d376d8b1666d
+size 269466288

all_checkpoints/stage1_07041727_2dataset/epoch=29.ckpt/checkpoint/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79bb71cb535f9037a708d5df0bdd2f1f7503f2863210a6e70d95e82e1252efd5
+size 269466352

all_checkpoints/stage1_07041727_2dataset/epoch=29.ckpt/checkpoint/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:69968693b4d1f1dc91388c21530f05303a247181a06837f0d1f7102aa1d08569
+size 269465648

all_checkpoints/stage1_07041727_2dataset/epoch=29.ckpt/checkpoint/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45daf7ab82636399b71a0e7f8e8f1e713eecccbc1b02231c42c37b2827efa557
+size 269466096

all_checkpoints/stage1_07041727_2dataset/epoch=29.ckpt/checkpoint/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3933fad3f64b1ec08005ff3f4b41707aec3bc8ad131e058fc01f7a7011b5063c
+size 269466352

all_checkpoints/stage1_07041727_2dataset/epoch=29.ckpt/checkpoint/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1972613b6bded397cb64a5cbfe6da779c1809df0dc8c91fff5d8a75aa82384b7
+size 269466096

all_checkpoints/stage1_07041727_2dataset/epoch=29.ckpt/checkpoint/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:19d448bb7539c16d523eaefd5f6417c700aed03f1a6c8395c99514462f86ee1e
+size 269465904

all_checkpoints/stage1_07041727_2dataset/epoch=29.ckpt/checkpoint/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b212a5b5020f57247311f70f59e51d7f724cad099891b5d172b38506ecc64642
+size 359448824

all_checkpoints/stage1_07041727_2dataset/epoch=29.ckpt/converted.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23721c244e955e6c1b8ef31b4e95492811f6b9bb605d28345b5cae3095393a5b
+size 718651684

all_checkpoints/stage1_07041727_2dataset/epoch=29.ckpt/latest

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47320987f9a49d5b00119b960f247a956773f57543982b8bfcb6da5bb3afd9ef
+size 10

all_checkpoints/stage1_07041727_2dataset/epoch=29.ckpt/zero_to_fp32.py

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:46497565ccf2b4a8b1f6f18c8341042f3749605a94335c81f69df1bd268af64f
+size 33272