Add files using upload-large-folder tool

ProtT3/all_checkpoints/stage1_06282348_ddp/wandb/debug-internal.log

@@ -0,0 +1,63 @@
+{"time":"2025-06-29T00:07:02.130913564+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.11","symlink path":"all_checkpoints/stage1_06282348_ddp/wandb/run-20250629_000702-rypk39yq/logs/debug-core.log"}
+{"time":"2025-06-29T00:07:16.339720801+08:00","level":"INFO","msg":"created new stream","id":"rypk39yq"}
+{"time":"2025-06-29T00:07:16.340562919+08:00","level":"INFO","msg":"stream: started","id":"rypk39yq"}
+{"time":"2025-06-29T00:07:16.340584288+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"rypk39yq"}
+{"time":"2025-06-29T00:07:16.340617888+08:00","level":"INFO","msg":"sender: started","stream_id":"rypk39yq"}
+{"time":"2025-06-29T00:07:16.340654242+08:00","level":"INFO","msg":"handler: started","stream_id":"rypk39yq"}
+{"time":"2025-06-29T00:07:28.033909694+08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2025-06-29T00:12:24.114755958+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06282348_ddp/rypk39yq/file_stream\": read tcp 10.1.6.17:52688->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-29T00:15:17.682707235+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06282348_ddp/rypk39yq/file_stream\": read tcp 10.1.6.17:43992->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-29T00:16:13.20335199+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:16:45.525802023+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:17:19.98711773+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:18:06.642780387+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06282348_ddp/rypk39yq/file_stream\": read tcp 10.1.6.17:36080->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-06-29T00:22:43.123257688+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06282348_ddp/rypk39yq/file_stream\": read tcp 10.1.6.17:52664->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-06-29T00:26:08.434737599+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06282348_ddp/rypk39yq/file_stream\": read tcp 10.1.6.17:42534->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-06-29T00:27:44.454100719+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06282348_ddp/rypk39yq/file_stream\": read tcp 10.1.6.17:53006->104.21.20.172:443: read: connection reset by peer"}
+{"time":"2025-06-29T00:29:13.211268181+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:29:45.68436365+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:30:19.759580601+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:30:33.650730605+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06282348_ddp/rypk39yq/file_stream\": read tcp 10.1.6.17:38754->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-06-29T00:30:58.011093426+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:34:39.922752645+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06282348_ddp/rypk39yq/file_stream\": read tcp 10.1.6.17:35350->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-06-29T00:36:41.88529828+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06282348_ddp/rypk39yq/file_stream\": unexpected EOF"}
+{"time":"2025-06-29T00:37:20.878368218+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06282348_ddp/rypk39yq/file_stream\": read tcp 10.1.6.17:46470->104.21.20.172:443: read: connection reset by peer"}
+{"time":"2025-06-29T00:38:49.414424011+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06282348_ddp/rypk39yq/file_stream\": unexpected EOF"}
+{"time":"2025-06-29T00:38:58.216757113+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:39:20.141003198+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06282348_ddp/rypk39yq/file_stream\": read tcp 10.1.6.17:53708->104.21.20.172:443: read: connection reset by peer"}
+{"time":"2025-06-29T00:41:33.299264534+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06282348_ddp/rypk39yq/file_stream\": read tcp 10.1.6.17:44198->104.21.20.172:443: read: connection reset by peer"}
+{"time":"2025-06-29T00:47:37.138754922+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06282348_ddp/rypk39yq/file_stream\": read tcp 10.1.6.17:39138->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-06-29T00:54:28.224811124+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:55:15.429710397+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06282348_ddp/rypk39yq/file_stream\": read tcp 10.1.6.17:49584->104.21.20.172:443: read: connection reset by peer"}
+{"time":"2025-06-29T00:55:36.251525534+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06282348_ddp/rypk39yq/file_stream\": read tcp 10.1.6.17:55184->104.21.20.172:443: read: connection reset by peer"}
+{"time":"2025-06-29T00:56:12.092902722+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06282348_ddp/rypk39yq/file_stream\": http2: client conn is closed"}
+{"time":"2025-06-29T00:59:32.604209299+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06282348_ddp/rypk39yq/file_stream\": read tcp 10.1.6.17:44582->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-06-29T01:00:43.231046844+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T01:05:28.234577388+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T01:06:00.428439859+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T01:06:35.403033399+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T01:07:13.835463934+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T01:12:30.014897464+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06282348_ddp/rypk39yq/file_stream\": unexpected EOF"}
+{"time":"2025-06-29T01:14:58.239397356+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T01:15:30.658073848+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T01:16:05.133874663+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T01:16:43.256922452+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T01:17:07.122753765+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06282348_ddp/rypk39yq/file_stream\": read tcp 10.1.6.17:42208->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-06-29T01:17:31.631854783+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T01:18:38.479583401+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T01:20:08.481626584+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T01:21:38.483904393+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T01:22:09.185192206+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06282348_ddp/rypk39yq/file_stream\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T01:28:06.578759778+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06282348_ddp/rypk39yq/file_stream\": read tcp 10.1.6.17:39408->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-06-29T02:00:40.766530394+08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.bandw.top/graphql","body":"error code: 502"}
+{"time":"2025-06-29T08:45:43.611887283+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T08:45:55.061157169+08:00","level":"INFO","msg":"api: retrying HTTP error","status":520,"url":"https://api.bandw.top/files/gia0603yucca/stage1_06282348_ddp/rypk39yq/file_stream","body":"error code: 520"}
+{"time":"2025-06-29T08:51:23.638432293+08:00","level":"INFO","msg":"api: retrying HTTP error","status":524,"url":"https://api.bandw.top/files/gia0603yucca/stage1_06282348_ddp/rypk39yq/file_stream","body":"error code: 524"}
+{"time":"2025-06-29T10:16:08.309722526+08:00","level":"INFO","msg":"stream: closing","id":"rypk39yq"}
+{"time":"2025-06-29T10:16:08.309813211+08:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2025-06-29T10:16:08.311047133+08:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2025-06-29T10:16:10.887637294+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-06-29T10:16:11.831362524+08:00","level":"INFO","msg":"handler: closed","stream_id":"rypk39yq"}
+{"time":"2025-06-29T10:16:11.831401295+08:00","level":"INFO","msg":"sender: closed","stream_id":"rypk39yq"}
+{"time":"2025-06-29T10:16:11.831391+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"rypk39yq"}
+{"time":"2025-06-29T10:16:11.835883161+08:00","level":"INFO","msg":"stream: closed","id":"rypk39yq"}

ProtT3/all_checkpoints/stage1_06282348_ddp/wandb/debug.log

@@ -0,0 +1,23 @@
+2025-06-29 00:07:02,120 INFO    MainThread:938398 [wandb_setup.py:_flush():70] Current SDK version is 0.19.11
+2025-06-29 00:07:02,120 INFO    MainThread:938398 [wandb_setup.py:_flush():70] Configure stats pid to 938398
+2025-06-29 00:07:02,120 INFO    MainThread:938398 [wandb_setup.py:_flush():70] Loading settings from /root/.config/wandb/settings
+2025-06-29 00:07:02,120 INFO    MainThread:938398 [wandb_setup.py:_flush():70] Loading settings from /nas/shared/kilab/wangyujia/ProtT3/wandb/settings
+2025-06-29 00:07:02,120 INFO    MainThread:938398 [wandb_setup.py:_flush():70] Loading settings from environment variables
+2025-06-29 00:07:02,120 INFO    MainThread:938398 [wandb_init.py:setup_run_log_directory():724] Logging user logs to ./all_checkpoints/stage1_06282348_ddp/wandb/run-20250629_000702-rypk39yq/logs/debug.log
+2025-06-29 00:07:02,120 INFO    MainThread:938398 [wandb_init.py:setup_run_log_directory():725] Logging internal logs to ./all_checkpoints/stage1_06282348_ddp/wandb/run-20250629_000702-rypk39yq/logs/debug-internal.log
+2025-06-29 00:07:02,120 INFO    MainThread:938398 [wandb_init.py:init():852] calling init triggers
+2025-06-29 00:07:02,120 INFO    MainThread:938398 [wandb_init.py:init():857] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-06-29 00:07:02,120 INFO    MainThread:938398 [wandb_init.py:init():893] starting backend
+2025-06-29 00:07:02,120 INFO    MainThread:938398 [wandb_init.py:init():897] sending inform_init request
+2025-06-29 00:07:02,122 INFO    MainThread:938398 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-06-29 00:07:02,125 INFO    MainThread:938398 [wandb_init.py:init():907] backend started and connected
+2025-06-29 00:07:02,126 INFO    MainThread:938398 [wandb_init.py:init():1005] updated telemetry
+2025-06-29 00:07:02,129 INFO    MainThread:938398 [wandb_init.py:init():1029] communicating run to backend with 90.0 second timeout
+2025-06-29 00:07:27,982 INFO    MainThread:938398 [wandb_init.py:init():1104] starting run threads in backend
+2025-06-29 00:07:28,171 INFO    MainThread:938398 [wandb_run.py:_console_start():2573] atexit reg
+2025-06-29 00:07:28,172 INFO    MainThread:938398 [wandb_run.py:_redirect():2421] redirect: wrap_raw
+2025-06-29 00:07:28,176 INFO    MainThread:938398 [wandb_run.py:_redirect():2490] Wrapping output streams.
+2025-06-29 00:07:28,176 INFO    MainThread:938398 [wandb_run.py:_redirect():2513] Redirects installed.
+2025-06-29 00:07:28,177 INFO    MainThread:938398 [wandb_init.py:init():1150] run started, returning control to user process
+2025-06-29 10:16:08,240 INFO    MsgRouterThr:938398 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles.

ProtT3/all_checkpoints/stage1_06282348_ddp/wandb/run-20250628_235150-ln8ma2mo/logs/debug-internal.log

@@ -0,0 +1,82 @@
+{"time":"2025-06-28T23:51:50.395952363+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.11","symlink path":"all_checkpoints/stage1_06282348_ddp/wandb/run-20250628_235150-ln8ma2mo/logs/debug-core.log"}
+{"time":"2025-06-28T23:52:20.501357263+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-28T23:52:52.730097178+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-28T23:53:27.526291573+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-28T23:54:05.599297713+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-28T23:54:54.222254418+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-28T23:56:01.770405008+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-28T23:57:31.772851692+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-28T23:59:01.774763186+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:00:31.777124138+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:02:01.779769246+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:03:31.781963651+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:05:01.784626624+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:06:31.787236433+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:08:01.789516886+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:09:31.792010561+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:11:01.793420114+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:12:31.795510043+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:14:01.797888169+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:15:31.800223897+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:17:01.802475149+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:18:31.804423686+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:18:31.805466058+08:00","level":"ERROR","msg":"Failed to load features, feature will default to disabled","error":"api: failed sending: POST https://api.wandb.ai/graphql giving up after 21 attempt(s): Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:18:31.817218548+08:00","level":"INFO","msg":"created new stream","id":"ln8ma2mo"}
+{"time":"2025-06-29T00:18:31.817240068+08:00","level":"INFO","msg":"stream: started","id":"ln8ma2mo"}
+{"time":"2025-06-29T00:18:31.817280389+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"ln8ma2mo"}
+{"time":"2025-06-29T00:18:31.817301971+08:00","level":"INFO","msg":"sender: started","stream_id":"ln8ma2mo"}
+{"time":"2025-06-29T00:18:31.817292844+08:00","level":"INFO","msg":"handler: started","stream_id":"ln8ma2mo"}
+{"time":"2025-06-29T00:18:31.817927543+08:00","level":"INFO","msg":"stream: closing","id":"ln8ma2mo"}
+{"time":"2025-06-29T00:19:01.824830492+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:19:34.122683021+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:20:08.859081137+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:20:47.142810832+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:21:34.117938021+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:22:42.456861603+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:24:12.458977097+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:25:42.460628156+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:27:12.462982763+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:28:31.82404899+08:00","level":"WARN","msg":"sender: taking a long time","seconds":600.000158503,"work":"WorkRecord(*service_go_proto.Record_Run); Control(mailbox_slot:\"elq8to0u7ndt\" always_send:true connection_id:\"127.0.0.1:48626\")"}
+{"time":"2025-06-29T00:28:42.464486984+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:30:12.465370037+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:31:42.467621727+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:33:12.470149312+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:34:42.471834466+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:36:12.474312455+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:37:42.477105562+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:38:31.82690936+08:00","level":"WARN","msg":"sender: taking a long time","seconds":1200.003036845,"work":"WorkRecord(*service_go_proto.Record_Run); Control(mailbox_slot:\"elq8to0u7ndt\" always_send:true connection_id:\"127.0.0.1:48626\")"}
+{"time":"2025-06-29T00:39:12.479327707+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:40:42.479854506+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:42:12.482469938+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:43:42.484924406+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:45:12.487708397+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:45:12.48877363+08:00","level":"ERROR","msg":"send: sendRun: failed to update run state: api: failed sending: POST https://api.wandb.ai/graphql giving up after 21 attempt(s): Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:45:12.489160854+08:00","level":"ERROR","msg":"sender: upsertConfig: RunRecord is nil"}
+{"time":"2025-06-29T00:45:12.489257088+08:00","level":"INFO","msg":"sender: succeeded after taking longer than expected","seconds":1600.665365029,"work":"WorkRecord(*service_go_proto.Record_Run); Control(mailbox_slot:\"elq8to0u7ndt\" always_send:true connection_id:\"127.0.0.1:48626\")"}
+{"time":"2025-06-29T00:45:42.54537918+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:46:14.577405421+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:46:49.560523299+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:47:29.044951023+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:48:18.616524778+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:49:24.200671698+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:50:54.203239638+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:52:24.20538484+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:53:54.207195938+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:55:24.209241621+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:56:54.212135665+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:58:24.214809507+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:59:54.217111858+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T01:01:24.219045106+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T01:02:54.221102245+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T01:04:24.22398357+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T01:05:54.226547409+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T01:07:24.228484779+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T01:08:54.231281118+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T01:10:24.233099781+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T01:11:54.236192516+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T01:11:54.236277338+08:00","level":"ERROR","msg":"runfiles: CreateRunFiles returned error: api: failed sending: POST https://api.wandb.ai/graphql giving up after 21 attempt(s): Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T01:11:54.236659368+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-06-29T01:11:54.239839801+08:00","level":"INFO","msg":"handler: closed","stream_id":"ln8ma2mo"}
+{"time":"2025-06-29T01:11:54.239896009+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"ln8ma2mo"}
+{"time":"2025-06-29T01:11:54.239911685+08:00","level":"INFO","msg":"sender: closed","stream_id":"ln8ma2mo"}
+{"time":"2025-06-29T01:11:54.242908101+08:00","level":"INFO","msg":"stream: closed","id":"ln8ma2mo"}

ProtT3/all_checkpoints/stage1_06282348_ddp/wandb/run-20250628_235150-ln8ma2mo/logs/debug.log

@@ -0,0 +1,94 @@
+2025-06-28 23:51:50,386 INFO    MainThread:906089 [wandb_setup.py:_flush():70] Current SDK version is 0.19.11
+2025-06-28 23:51:50,386 INFO    MainThread:906089 [wandb_setup.py:_flush():70] Configure stats pid to 906089
+2025-06-28 23:51:50,386 INFO    MainThread:906089 [wandb_setup.py:_flush():70] Loading settings from /root/.config/wandb/settings
+2025-06-28 23:51:50,386 INFO    MainThread:906089 [wandb_setup.py:_flush():70] Loading settings from /nas/shared/kilab/wangyujia/ProtT3/wandb/settings
+2025-06-28 23:51:50,386 INFO    MainThread:906089 [wandb_setup.py:_flush():70] Loading settings from environment variables
+2025-06-28 23:51:50,386 INFO    MainThread:906089 [wandb_init.py:setup_run_log_directory():724] Logging user logs to ./all_checkpoints/stage1_06282348_ddp/wandb/run-20250628_235150-ln8ma2mo/logs/debug.log
+2025-06-28 23:51:50,386 INFO    MainThread:906089 [wandb_init.py:setup_run_log_directory():725] Logging internal logs to ./all_checkpoints/stage1_06282348_ddp/wandb/run-20250628_235150-ln8ma2mo/logs/debug-internal.log
+2025-06-28 23:51:50,386 INFO    MainThread:906089 [wandb_init.py:init():852] calling init triggers
+2025-06-28 23:51:50,386 INFO    MainThread:906089 [wandb_init.py:init():857] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-06-28 23:51:50,386 INFO    MainThread:906089 [wandb_init.py:init():893] starting backend
+2025-06-28 23:51:50,386 INFO    MainThread:906089 [wandb_init.py:init():897] sending inform_init request
+2025-06-28 23:51:50,389 INFO    MainThread:906089 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-06-28 23:51:50,390 INFO    MainThread:906089 [wandb_init.py:init():907] backend started and connected
+2025-06-28 23:51:50,391 INFO    MainThread:906089 [wandb_init.py:init():1005] updated telemetry
+2025-06-28 23:51:50,393 INFO    MainThread:906089 [wandb_init.py:init():1029] communicating run to backend with 90.0 second timeout
+2025-06-28 23:51:59,762 WARNING MainThread:906089 [wandb_init.py:init():1681] [no run ID] interrupted
+Traceback (most recent call last):
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/wandb/sdk/wandb_init.py", line 1677, in init
+    return wi.init(run_settings, run_config, run_printer)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/wandb/sdk/wandb_init.py", line 1055, in init
+    result = wait_with_progress(
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/wandb/sdk/mailbox/wait_with_progress.py", line 24, in wait_with_progress
+    return wait_all_with_progress(
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/wandb/sdk/mailbox/wait_with_progress.py", line 87, in wait_all_with_progress
+    return asyncio_compat.run(progress_loop_with_timeout)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/wandb/sdk/lib/asyncio_compat.py", line 30, in run
+    return future.result()
+  File "/root/miniconda3/envs/protT3/lib/python3.10/concurrent/futures/_base.py", line 440, in result
+    self._condition.wait(timeout)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/threading.py", line 320, in wait
+    waiter.acquire()
+KeyboardInterrupt
+2025-06-28 23:52:00,349 INFO    MsgRouterThr:906089 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 2 handles.
+2025-06-28 23:52:21,541 INFO    Thread-3 (wrapped_target):906089 [retry.py:__call__():175] [no run ID] Retry attempt failed:
+Traceback (most recent call last):
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/urllib3/connection.py", line 198, in _new_conn
+    sock = connection.create_connection(
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/urllib3/util/connection.py", line 85, in create_connection
+    raise err
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/urllib3/util/connection.py", line 73, in create_connection
+    sock.connect(sa)
+TimeoutError: timed out
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/urllib3/connectionpool.py", line 787, in urlopen
+    response = self._make_request(
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/urllib3/connectionpool.py", line 488, in _make_request
+    raise new_e
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/urllib3/connectionpool.py", line 464, in _make_request
+    self._validate_conn(conn)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/urllib3/connectionpool.py", line 1093, in _validate_conn
+    conn.connect()
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/urllib3/connection.py", line 704, in connect
+    self.sock = sock = self._new_conn()
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/urllib3/connection.py", line 207, in _new_conn
+    raise ConnectTimeoutError(
+urllib3.exceptions.ConnectTimeoutError: (<urllib3.connection.HTTPSConnection object at 0x7f9f447cebc0>, 'Connection to api.wandb.ai timed out. (connect timeout=20)')
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/requests/adapters.py", line 667, in send
+    resp = conn.urlopen(
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/urllib3/connectionpool.py", line 841, in urlopen
+    retries = retries.increment(
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/urllib3/util/retry.py", line 519, in increment
+    raise MaxRetryError(_pool, url, reason) from reason  # type: ignore[arg-type]
+urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='api.wandb.ai', port=443): Max retries exceeded with url: /graphql (Caused by ConnectTimeoutError(<urllib3.connection.HTTPSConnection object at 0x7f9f447cebc0>, 'Connection to api.wandb.ai timed out. (connect timeout=20)'))
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/wandb/sdk/lib/retry.py", line 134, in __call__
+    result = self._call_fn(*args, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/wandb/sdk/internal/internal_api.py", line 398, in execute
+    return self.client.execute(*args, **kwargs)  # type: ignore
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/wandb/vendor/gql-0.2.0/wandb_gql/client.py", line 52, in execute
+    result = self._get_result(document, *args, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/wandb/vendor/gql-0.2.0/wandb_gql/client.py", line 60, in _get_result
+    return self.transport.execute(document, *args, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/wandb/sdk/lib/gql_request.py", line 58, in execute
+    request = self.session.post(self.url, **post_args)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/requests/sessions.py", line 637, in post
+    return self.request("POST", url, data=data, json=json, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/requests/sessions.py", line 589, in request
+    resp = self.send(prep, **send_kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/requests/sessions.py", line 703, in send
+    r = adapter.send(request, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/requests/adapters.py", line 688, in send
+    raise ConnectTimeout(e, request=request)
+requests.exceptions.ConnectTimeout: HTTPSConnectionPool(host='api.wandb.ai', port=443): Max retries exceeded with url: /graphql (Caused by ConnectTimeoutError(<urllib3.connection.HTTPSConnection object at 0x7f9f447cebc0>, 'Connection to api.wandb.ai timed out. (connect timeout=20)'))

ProtT3/all_checkpoints/stage1_06282348_ddp/wandb/run-20250628_235330-mz3ej8ig/logs/debug-internal.log

@@ -0,0 +1,82 @@
+{"time":"2025-06-28T23:53:30.951932242+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.11","symlink path":"all_checkpoints/stage1_06282348_ddp/wandb/run-20250628_235330-mz3ej8ig/logs/debug-core.log"}
+{"time":"2025-06-28T23:54:01.056320508+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-28T23:54:33.136125443+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-28T23:55:07.262654908+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-28T23:55:47.196366157+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-28T23:56:33.830440313+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-28T23:57:41.624410258+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-28T23:59:11.626386567+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:00:41.62885613+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:02:11.630480713+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:03:41.632896773+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:05:11.634787095+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:06:41.637469615+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:08:11.640346542+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:09:41.642435888+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:11:11.643824358+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:12:41.645847752+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:14:11.647375061+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:15:41.649750172+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:17:11.652403406+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:18:41.655408766+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:20:11.657578168+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:20:11.658626379+08:00","level":"ERROR","msg":"Failed to load features, feature will default to disabled","error":"api: failed sending: POST https://api.wandb.ai/graphql giving up after 21 attempt(s): Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:20:11.681430338+08:00","level":"INFO","msg":"created new stream","id":"mz3ej8ig"}
+{"time":"2025-06-29T00:20:11.681462583+08:00","level":"INFO","msg":"stream: started","id":"mz3ej8ig"}
+{"time":"2025-06-29T00:20:11.681507089+08:00","level":"INFO","msg":"sender: started","stream_id":"mz3ej8ig"}
+{"time":"2025-06-29T00:20:11.68149584+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"mz3ej8ig"}
+{"time":"2025-06-29T00:20:11.681547365+08:00","level":"INFO","msg":"handler: started","stream_id":"mz3ej8ig"}
+{"time":"2025-06-29T00:20:11.682347217+08:00","level":"INFO","msg":"stream: closing","id":"mz3ej8ig"}
+{"time":"2025-06-29T00:20:41.686911252+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:21:14.167970232+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:21:48.35484514+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:22:27.366864931+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:23:15.010485407+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:24:21.797181116+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:25:51.799191068+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:27:21.801584003+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:28:51.804025943+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:30:11.686396042+08:00","level":"WARN","msg":"sender: taking a long time","seconds":600.000441033,"work":"WorkRecord(*service_go_proto.Record_Run); Control(mailbox_slot:\"5dtwv5qkpd53\" always_send:true connection_id:\"127.0.0.1:33438\")"}
+{"time":"2025-06-29T00:30:21.806550019+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:31:51.807720521+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:33:21.80993678+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:34:51.811863783+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:36:21.813546407+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:37:51.815079734+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:39:21.816778971+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:40:11.688234649+08:00","level":"WARN","msg":"sender: taking a long time","seconds":1200.002273841,"work":"WorkRecord(*service_go_proto.Record_Run); Control(mailbox_slot:\"5dtwv5qkpd53\" always_send:true connection_id:\"127.0.0.1:33438\")"}
+{"time":"2025-06-29T00:40:51.81937773+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:42:21.820944763+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:43:51.823708778+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:45:21.825965407+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:46:51.827707527+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:46:51.828793803+08:00","level":"ERROR","msg":"send: sendRun: failed to update run state: api: failed sending: POST https://api.wandb.ai/graphql giving up after 21 attempt(s): Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:46:51.829193166+08:00","level":"ERROR","msg":"sender: upsertConfig: RunRecord is nil"}
+{"time":"2025-06-29T00:46:51.82925862+08:00","level":"INFO","msg":"sender: succeeded after taking longer than expected","seconds":1600.143330565,"work":"WorkRecord(*service_go_proto.Record_Run); Control(mailbox_slot:\"5dtwv5qkpd53\" always_send:true connection_id:\"127.0.0.1:33438\")"}
+{"time":"2025-06-29T00:47:21.884399165+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:47:54.221719208+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:48:28.514834678+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:49:08.340678437+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:49:57.755779717+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:51:02.131711303+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:52:32.133820589+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:54:02.13693015+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:55:32.139789811+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:57:02.14632236+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T00:58:32.148909001+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T01:00:02.154970223+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T01:01:32.157829588+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T01:03:02.159580228+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T01:04:32.161980354+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T01:06:02.163980101+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T01:07:32.166420035+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T01:09:02.168842314+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T01:10:32.171375938+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T01:12:02.17356805+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T01:13:32.174403974+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T01:13:32.175440214+08:00","level":"ERROR","msg":"runfiles: CreateRunFiles returned error: api: failed sending: POST https://api.wandb.ai/graphql giving up after 21 attempt(s): Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T01:13:32.175867374+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-06-29T01:13:32.175926547+08:00","level":"INFO","msg":"handler: closed","stream_id":"mz3ej8ig"}
+{"time":"2025-06-29T01:13:32.175943588+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"mz3ej8ig"}
+{"time":"2025-06-29T01:13:32.176007588+08:00","level":"INFO","msg":"sender: closed","stream_id":"mz3ej8ig"}
+{"time":"2025-06-29T01:13:32.180310341+08:00","level":"INFO","msg":"stream: closed","id":"mz3ej8ig"}

ProtT3/all_checkpoints/stage1_06282348_ddp/wandb/run-20250628_235330-mz3ej8ig/logs/debug.log

@@ -0,0 +1,94 @@
+2025-06-28 23:53:30,942 INFO    MainThread:907243 [wandb_setup.py:_flush():70] Current SDK version is 0.19.11
+2025-06-28 23:53:30,942 INFO    MainThread:907243 [wandb_setup.py:_flush():70] Configure stats pid to 907243
+2025-06-28 23:53:30,942 INFO    MainThread:907243 [wandb_setup.py:_flush():70] Loading settings from /root/.config/wandb/settings
+2025-06-28 23:53:30,942 INFO    MainThread:907243 [wandb_setup.py:_flush():70] Loading settings from /nas/shared/kilab/wangyujia/ProtT3/wandb/settings
+2025-06-28 23:53:30,942 INFO    MainThread:907243 [wandb_setup.py:_flush():70] Loading settings from environment variables
+2025-06-28 23:53:30,942 INFO    MainThread:907243 [wandb_init.py:setup_run_log_directory():724] Logging user logs to ./all_checkpoints/stage1_06282348_ddp/wandb/run-20250628_235330-mz3ej8ig/logs/debug.log
+2025-06-28 23:53:30,942 INFO    MainThread:907243 [wandb_init.py:setup_run_log_directory():725] Logging internal logs to ./all_checkpoints/stage1_06282348_ddp/wandb/run-20250628_235330-mz3ej8ig/logs/debug-internal.log
+2025-06-28 23:53:30,942 INFO    MainThread:907243 [wandb_init.py:init():852] calling init triggers
+2025-06-28 23:53:30,942 INFO    MainThread:907243 [wandb_init.py:init():857] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-06-28 23:53:30,943 INFO    MainThread:907243 [wandb_init.py:init():893] starting backend
+2025-06-28 23:53:30,943 INFO    MainThread:907243 [wandb_init.py:init():897] sending inform_init request
+2025-06-28 23:53:30,944 INFO    MainThread:907243 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-06-28 23:53:30,946 INFO    MainThread:907243 [wandb_init.py:init():907] backend started and connected
+2025-06-28 23:53:30,950 INFO    MainThread:907243 [wandb_init.py:init():1005] updated telemetry
+2025-06-28 23:53:30,953 INFO    MainThread:907243 [wandb_init.py:init():1029] communicating run to backend with 90.0 second timeout
+2025-06-28 23:54:01,206 WARNING MainThread:907243 [wandb_init.py:init():1681] [no run ID] interrupted
+Traceback (most recent call last):
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/wandb/sdk/wandb_init.py", line 1677, in init
+    return wi.init(run_settings, run_config, run_printer)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/wandb/sdk/wandb_init.py", line 1055, in init
+    result = wait_with_progress(
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/wandb/sdk/mailbox/wait_with_progress.py", line 24, in wait_with_progress
+    return wait_all_with_progress(
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/wandb/sdk/mailbox/wait_with_progress.py", line 87, in wait_all_with_progress
+    return asyncio_compat.run(progress_loop_with_timeout)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/wandb/sdk/lib/asyncio_compat.py", line 30, in run
+    return future.result()
+  File "/root/miniconda3/envs/protT3/lib/python3.10/concurrent/futures/_base.py", line 440, in result
+    self._condition.wait(timeout)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/threading.py", line 320, in wait
+    waiter.acquire()
+KeyboardInterrupt
+2025-06-28 23:54:01,916 INFO    MsgRouterThr:907243 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 2 handles.
+2025-06-28 23:54:02,523 INFO    Thread-3 (wrapped_target):907243 [retry.py:__call__():175] [no run ID] Retry attempt failed:
+Traceback (most recent call last):
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/urllib3/connection.py", line 198, in _new_conn
+    sock = connection.create_connection(
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/urllib3/util/connection.py", line 85, in create_connection
+    raise err
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/urllib3/util/connection.py", line 73, in create_connection
+    sock.connect(sa)
+TimeoutError: timed out
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/urllib3/connectionpool.py", line 787, in urlopen
+    response = self._make_request(
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/urllib3/connectionpool.py", line 488, in _make_request
+    raise new_e
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/urllib3/connectionpool.py", line 464, in _make_request
+    self._validate_conn(conn)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/urllib3/connectionpool.py", line 1093, in _validate_conn
+    conn.connect()
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/urllib3/connection.py", line 704, in connect
+    self.sock = sock = self._new_conn()
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/urllib3/connection.py", line 207, in _new_conn
+    raise ConnectTimeoutError(
+urllib3.exceptions.ConnectTimeoutError: (<urllib3.connection.HTTPSConnection object at 0x7f87fd2d6350>, 'Connection to api.wandb.ai timed out. (connect timeout=20)')
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/requests/adapters.py", line 667, in send
+    resp = conn.urlopen(
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/urllib3/connectionpool.py", line 841, in urlopen
+    retries = retries.increment(
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/urllib3/util/retry.py", line 519, in increment
+    raise MaxRetryError(_pool, url, reason) from reason  # type: ignore[arg-type]
+urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='api.wandb.ai', port=443): Max retries exceeded with url: /graphql (Caused by ConnectTimeoutError(<urllib3.connection.HTTPSConnection object at 0x7f87fd2d6350>, 'Connection to api.wandb.ai timed out. (connect timeout=20)'))
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/wandb/sdk/lib/retry.py", line 134, in __call__
+    result = self._call_fn(*args, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/wandb/sdk/internal/internal_api.py", line 398, in execute
+    return self.client.execute(*args, **kwargs)  # type: ignore
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/wandb/vendor/gql-0.2.0/wandb_gql/client.py", line 52, in execute
+    result = self._get_result(document, *args, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/wandb/vendor/gql-0.2.0/wandb_gql/client.py", line 60, in _get_result
+    return self.transport.execute(document, *args, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/wandb/sdk/lib/gql_request.py", line 58, in execute
+    request = self.session.post(self.url, **post_args)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/requests/sessions.py", line 637, in post
+    return self.request("POST", url, data=data, json=json, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/requests/sessions.py", line 589, in request
+    resp = self.send(prep, **send_kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/requests/sessions.py", line 703, in send
+    r = adapter.send(request, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/requests/adapters.py", line 688, in send
+    raise ConnectTimeout(e, request=request)
+requests.exceptions.ConnectTimeout: HTTPSConnectionPool(host='api.wandb.ai', port=443): Max retries exceeded with url: /graphql (Caused by ConnectTimeoutError(<urllib3.connection.HTTPSConnection object at 0x7f87fd2d6350>, 'Connection to api.wandb.ai timed out. (connect timeout=20)'))

ProtT3/all_checkpoints/stage1_06282348_ddp/wandb/run-20250628_235448-d21a8n96/files/config.yaml

@@ -0,0 +1,129 @@
+_wandb:
+    value:
+        cli_version: 0.19.11
+        m:
+            - "1": trainer/global_step
+              "6":
+                - 3
+              "7": []
+        python_version: 3.10.0
+        t:
+            "1":
+                - 1
+                - 5
+                - 9
+                - 11
+                - 33
+                - 41
+                - 49
+                - 53
+                - 55
+                - 63
+                - 103
+            "2":
+                - 1
+                - 5
+                - 9
+                - 11
+                - 33
+                - 41
+                - 49
+                - 53
+                - 55
+                - 63
+                - 103
+            "3":
+                - 7
+                - 23
+                - 33
+                - 55
+                - 66
+            "4": 3.10.0
+            "5": 0.19.11
+            "6": 4.52.3
+            "8":
+                - 5
+            "12": 0.19.11
+            "13": linux-x86_64
+accelerator:
+    value: gpu
+batch_size:
+    value: 96
+bert_hidden_dim:
+    value: 768
+bert_name:
+    value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft
+check_val_every_n_epoch:
+    value: 1
+cross_attention_freq:
+    value: 2
+devices:
+    value: 0,1,2,3,4,5,6,7
+filename:
+    value: stage1_06282348_ddp
+init_checkpoint:
+    value: ""
+init_lr:
+    value: 0.0001
+lm:
+    value: true
+load_4bit:
+    value: false
+lr_decay_rate:
+    value: 0.9
+match_batch_size:
+    value: 64
+max_epochs:
+    value: 20
+min_lr:
+    value: 1e-05
+mix_dataset:
+    value: true
+mode:
+    value: train
+num_query_token:
+    value: 8
+num_workers:
+    value: 8
+plm_name:
+    value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m
+plm_tune:
+    value: freeze
+pool_size:
+    value: 0
+precision:
+    value: bf16-mixed
+projection_dim:
+    value: 256
+prot_aug:
+    value: None
+prot_max_len:
+    value: 1024
+ptm:
+    value: true
+rerank_cand_num:
+    value: 128
+retrieval_eval_epoch:
+    value: 10
+root:
+    value: data
+save_every_n_epochs:
+    value: 5
+scheduler:
+    value: linear_warmup_cosine_lr
+seed:
+    value: 42
+strategy:
+    value: ddp
+temperature:
+    value: 0.1
+text_max_len:
+    value: 128
+use_wandb_logger:
+    value: true
+warmup_lr:
+    value: 1e-06
+warmup_steps:
+    value: 1000
+weight_decay:
+    value: 0.05

ProtT3/all_checkpoints/stage1_06282348_ddp/wandb/run-20250628_235448-d21a8n96/files/output.log

@@ -0,0 +1,21 @@
+W0629 00:01:34.010646 908035 site-packages/torch/multiprocessing/spawn.py:169] Terminating process 908727 via signal SIGTERM
+W0629 00:01:34.011594 908035 site-packages/torch/multiprocessing/spawn.py:169] Terminating process 908857 via signal SIGTERM
+W0629 00:01:34.011932 908035 site-packages/torch/multiprocessing/spawn.py:169] Terminating process 909000 via signal SIGTERM
+W0629 00:01:34.012163 908035 site-packages/torch/multiprocessing/spawn.py:169] Terminating process 909139 via signal SIGTERM
+W0629 00:01:34.012400 908035 site-packages/torch/multiprocessing/spawn.py:169] Terminating process 909288 via signal SIGTERM
+W0629 00:01:34.012614 908035 site-packages/torch/multiprocessing/spawn.py:169] Terminating process 909427 via signal SIGTERM
+W0629 00:01:34.012998 908035 site-packages/torch/multiprocessing/spawn.py:169] Terminating process 909567 via signal SIGTERM
+Traceback (most recent call last):
+  File "/nas/shared/kilab/wangyujia/ProtT3/stage1.py", line 135, in <module>
+    main(args)
+  File "/nas/shared/kilab/wangyujia/ProtT3/stage1.py", line 101, in main
+    trainer.fit(model, datamodule=dm)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 561, in fit
+    call._call_and_handle_interrupt(
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 47, in _call_and_handle_interrupt
+    return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/strategies/launchers/multiprocessing.py", line 144, in launch
+    while not process_context.join():
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/multiprocessing/spawn.py", line 215, in join
+    raise ProcessRaisedException(msg, error_index, failed_process.pid)
+torch.multiprocessing.spawn.ProcessRaisedException: It looks like your LightningModule has parameters that were not used in producing the loss returned by training_step. If this is intentional, you must enable the detection of unused parameters in DDP, either by setting the string value `strategy='ddp_find_unused_parameters_true'` or by setting the flag in the strategy with `strategy=DDPStrategy(find_unused_parameters=True)`.

ProtT3/all_checkpoints/stage1_06282348_ddp/wandb/run-20250628_235448-d21a8n96/files/requirements.txt

@@ -0,0 +1,225 @@
+stack-data==0.6.3
+yarl==1.20.0
+setuptools==78.1.1
+cloudpathlib==0.21.1
+pytz==2025.2
+nvidia-cufile-cu12==1.11.1.6
+shellingham==1.5.4
+nvidia-cusolver-cu12==11.6.1.9
+Jinja2==3.1.6
+pycocotools==2.0.8
+pandas==2.2.3
+scipy==1.15.3
+tenacity==9.1.2
+lightning-utilities==0.14.3
+cfgv==3.4.0
+hf-xet==1.1.2
+platformdirs==4.3.8
+smart-open==7.1.0
+text-unidecode==1.3
+nvidia-cublas-cu12==12.4.5.8
+omegaconf==2.3.0
+referencing==0.36.2
+mdurl==0.1.2
+gitdb==4.0.12
+identify==2.6.12
+ipython==8.36.0
+spacy-loggers==1.0.5
+distlib==0.3.9
+typing-inspection==0.4.1
+antlr4-python3-runtime==4.9.3
+multidict==6.4.4
+nvidia-curand-cu12==10.3.5.147
+prompt_toolkit==3.0.51
+Pygments==2.19.1
+numpy==2.2.6
+decord==0.6.0
+srsly==2.5.1
+watchdog==6.0.0
+pure_eval==0.2.3
+virtualenv==20.31.2
+altair==5.5.0
+matplotlib-inline==0.1.7
+bleach==6.2.0
+exceptiongroup==1.3.0
+fairscale==0.4.4
+confection==0.1.5
+fonttools==4.58.0
+nvidia-cuda-nvrtc-cu12==12.4.127
+ptyprocess==0.7.0
+pytorch-lightning==2.5.1.post0
+nodeenv==1.9.1
+nvidia-cudnn-cu12==9.1.0.70
+requests==2.32.3
+marisa-trie==1.2.1
+cachetools==5.5.2
+matplotlib==3.10.3
+typing_extensions==4.13.2
+asttokens==3.0.0
+torch==2.6.0
+PyYAML==6.0.2
+tifffile==2025.5.10
+spacy==3.8.7
+braceexpand==0.1.7
+plotly==6.1.1
+attrs==25.3.0
+py-cpuinfo==9.0.0
+frozenlist==1.6.0
+catalogue==2.0.10
+nvidia-cusparselt-cu12==0.6.2
+traitlets==5.14.3
+annotated-types==0.7.0
+language_data==1.3.0
+thinc==8.3.6
+imageio==2.37.0
+nvidia-cuda-runtime-cu12==12.4.127
+certifi==2025.4.26
+smmap==5.0.2
+python-magic==0.4.27
+triton==3.2.0
+weasel==0.4.1
+async-timeout==5.0.1
+wcwidth==0.2.13
+pillow==11.2.1
+torchmetrics==1.7.1
+kaggle==1.7.4.5
+regex==2024.11.6
+aiosignal==1.3.2
+nvidia-cusparse-cu12==12.3.1.170
+scikit-image==0.25.2
+nvidia-nvtx-cu12==12.4.127
+opendatasets==0.1.22
+iopath==0.1.10
+pyparsing==3.2.3
+portalocker==3.1.1
+executing==2.2.0
+contexttimer==0.3.3
+lazy_loader==0.4
+wrapt==1.17.2
+webdataset==0.2.111
+blis==1.3.0
+idna==3.10
+timm==0.4.12
+einops==0.8.1
+packaging==24.2
+decorator==5.2.1
+filelock==3.18.0
+python-slugify==8.0.4
+cycler==0.12.1
+charset-normalizer==3.4.2
+pydantic==2.11.5
+pydeck==0.9.1
+tzdata==2025.2
+jedi==0.19.2
+aiohappyeyeballs==2.6.1
+nvidia-nvjitlink-cu12==12.4.127
+salesforce-lavis==1.0.2
+parso==0.8.4
+nvidia-nccl-cu12==2.21.5
+toml==0.10.2
+python-dateutil==2.9.0.post0
+rich==14.0.0
+tqdm==4.67.1
+rpds-py==0.25.1
+opencv-python-headless==4.5.5.64
+tornado==6.5.1
+propcache==0.3.1
+webencodings==0.5.1
+murmurhash==1.0.13
+contourpy==1.3.2
+joblib==1.5.1
+networkx==3.4.2
+six==1.17.0
+markdown-it-py==3.0.0
+nvidia-cuda-cupti-cu12==12.4.127
+msgpack==1.1.0
+sentencepiece==0.2.0
+cymem==2.0.11
+nvidia-cufft-cu12==11.2.1.3
+absl-py==2.2.2
+hjson==3.1.0
+mpmath==1.3.0
+pydantic_core==2.33.2
+psutil==7.0.0
+nvidia-ml-py==12.575.51
+pyarrow==20.0.0
+kiwisolver==1.4.8
+sympy==1.13.1
+ninja==1.11.1.4
+rouge_score==0.1.2
+deepspeed==0.16.10+b666844f
+spacy-legacy==3.0.12
+pycocoevalcap==1.2
+pexpect==4.9.0
+ftfy==6.3.1
+protobuf==6.31.0
+urllib3==2.4.0
+wheel==0.45.1
+nltk==3.9.1
+streamlit==1.45.1
+wasabi==1.1.3
+pre_commit==4.2.0
+safetensors==0.5.3
+jsonschema-specifications==2025.4.1
+langcodes==3.5.0
+GitPython==3.1.44
+blinker==1.9.0
+torchvision==0.21.0
+MarkupSafe==3.0.2
+dill==0.3.8
+yacs==0.1.8
+pathlib==1.0.1
+scikit-learn==1.6.1
+cffi==1.17.1
+pycparser==2.22
+flash-attn==2.7.1.post1
+cryptography==45.0.3
+pycryptodome==3.23.0
+cheroot==10.0.1
+more-itertools==10.7.0
+setproctitle==1.3.6
+delta-center-client==0.0.4
+jmespath==0.10.0
+xxhash==3.5.0
+pip==25.1.1
+aliyun-python-sdk-core==2.16.0
+jaraco.functools==4.1.0
+bigmodelvis==0.0.1
+aiohttp==3.12.2
+multiprocess==0.70.16
+opendelta==0.3.2
+docker-pycreds==0.4.0
+threadpoolctl==3.6.0
+click==8.2.1
+oss2==2.15.0
+crcmod==1.7
+transformers==4.52.3
+datasets==3.6.0
+jsonschema==4.24.0
+opencv-python==4.11.0.86
+wandb==0.19.11
+fsspec==2025.3.0
+tokenizers==0.21.1
+sentry-sdk==2.29.1
+preshed==3.0.10
+aliyun-python-sdk-kms==2.16.5
+huggingface-hub==0.32.1
+typer==0.16.0
+narwhals==1.41.0
+web.py==0.62
+autocommand==2.2.2
+importlib_metadata==8.0.0
+zipp==3.19.2
+jaraco.context==5.3.0
+typeguard==4.3.0
+jaraco.collections==5.1.0
+typing_extensions==4.12.2
+backports.tarfile==1.2.0
+jaraco.functools==4.0.1
+more-itertools==10.3.0
+platformdirs==4.2.2
+packaging==24.2
+tomli==2.0.1
+jaraco.text==3.12.1
+wheel==0.45.1
+inflect==7.3.1

ProtT3/all_checkpoints/stage1_06282348_ddp/wandb/run-20250628_235448-d21a8n96/files/wandb-metadata.json

@@ -0,0 +1,99 @@
+{
+  "os": "Linux-5.10.134-008.16.kangaroo.al8.x86_64-x86_64-with-glibc2.35",
+  "python": "CPython 3.10.0",
+  "startedAt": "2025-06-28T15:54:48.951303Z",
+  "args": [
+    "--devices",
+    "0,1,2,3,4,5,6,7",
+    "--mode",
+    "train",
+    "--filename",
+    "stage1_06282348_ddp",
+    "--num_query_token",
+    "8",
+    "--plm_name",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m",
+    "--bert_name",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft",
+    "--save_every_n_epochs",
+    "5",
+    "--max_epochs",
+    "20",
+    "--batch_size",
+    "96",
+    "--precision",
+    "bf16-mixed",
+    "--mix_dataset",
+    "--num_workers",
+    "8",
+    "--use_wandb_logger",
+    "--strategy",
+    "ddp"
+  ],
+  "program": "/nas/shared/kilab/wangyujia/ProtT3/stage1.py",
+  "codePath": "stage1.py",
+  "root": "./all_checkpoints/stage1_06282348_ddp/",
+  "host": "dsw-265304-57b7b77cbc-vwbwc",
+  "executable": "/root/miniconda3/envs/protT3/bin/python",
+  "codePathLocal": "stage1.py",
+  "cpu_count": 64,
+  "cpu_count_logical": 64,
+  "gpu": "NVIDIA A800-SXM4-80GB",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1623302262784",
+      "used": "1285226496"
+    }
+  },
+  "memory": {
+    "total": "549755813888"
+  },
+  "cpu": {
+    "count": 64,
+    "countLogical": 64
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    }
+  ],
+  "cudaVersion": "12.1"
+}

ProtT3/all_checkpoints/stage1_06282348_ddp/wandb/run-20250628_235448-d21a8n96/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":408}}

ProtT3/all_checkpoints/stage1_06282348_ddp/wandb/run-20250628_235448-d21a8n96/logs/debug-internal.log

@@ -0,0 +1,18 @@
+{"time":"2025-06-28T23:54:48.952903363+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.11","symlink path":"all_checkpoints/stage1_06282348_ddp/wandb/run-20250628_235448-d21a8n96/logs/debug-core.log"}
+{"time":"2025-06-28T23:55:08.706410301+08:00","level":"INFO","msg":"created new stream","id":"d21a8n96"}
+{"time":"2025-06-28T23:55:08.70719709+08:00","level":"INFO","msg":"stream: started","id":"d21a8n96"}
+{"time":"2025-06-28T23:55:08.707235941+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"d21a8n96"}
+{"time":"2025-06-28T23:55:08.707321639+08:00","level":"INFO","msg":"handler: started","stream_id":"d21a8n96"}
+{"time":"2025-06-28T23:55:08.707259857+08:00","level":"INFO","msg":"sender: started","stream_id":"d21a8n96"}
+{"time":"2025-06-28T23:55:15.734662691+08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2025-06-28T23:58:30.738393146+08:00","level":"ERROR","msg":"filestream: json decode error: context deadline exceeded (Client.Timeout or context cancellation while reading body)"}
+{"time":"2025-06-28T23:58:30.746811098+08:00","level":"ERROR","msg":"filestream: error closing response body: net/http: request canceled"}
+{"time":"2025-06-29T00:01:37.883490068+08:00","level":"INFO","msg":"stream: closing","id":"d21a8n96"}
+{"time":"2025-06-29T00:01:37.883589533+08:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2025-06-29T00:01:37.884424806+08:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2025-06-29T00:01:42.08475624+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-06-29T00:02:28.146748335+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06282348_ddp/d21a8n96/file_stream\": read tcp 10.1.6.17:45786->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-06-29T00:02:54.866501576+08:00","level":"INFO","msg":"handler: closed","stream_id":"d21a8n96"}
+{"time":"2025-06-29T00:02:54.86656561+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"d21a8n96"}
+{"time":"2025-06-29T00:02:54.86658917+08:00","level":"INFO","msg":"sender: closed","stream_id":"d21a8n96"}
+{"time":"2025-06-29T00:02:54.871635053+08:00","level":"INFO","msg":"stream: closed","id":"d21a8n96"}

ProtT3/all_checkpoints/stage1_06282348_ddp/wandb/run-20250628_235448-d21a8n96/logs/debug.log

@@ -0,0 +1,23 @@
+2025-06-28 23:54:48,944 INFO    MainThread:908035 [wandb_setup.py:_flush():70] Current SDK version is 0.19.11
+2025-06-28 23:54:48,944 INFO    MainThread:908035 [wandb_setup.py:_flush():70] Configure stats pid to 908035
+2025-06-28 23:54:48,944 INFO    MainThread:908035 [wandb_setup.py:_flush():70] Loading settings from /root/.config/wandb/settings
+2025-06-28 23:54:48,944 INFO    MainThread:908035 [wandb_setup.py:_flush():70] Loading settings from /nas/shared/kilab/wangyujia/ProtT3/wandb/settings
+2025-06-28 23:54:48,944 INFO    MainThread:908035 [wandb_setup.py:_flush():70] Loading settings from environment variables
+2025-06-28 23:54:48,944 INFO    MainThread:908035 [wandb_init.py:setup_run_log_directory():724] Logging user logs to ./all_checkpoints/stage1_06282348_ddp/wandb/run-20250628_235448-d21a8n96/logs/debug.log
+2025-06-28 23:54:48,944 INFO    MainThread:908035 [wandb_init.py:setup_run_log_directory():725] Logging internal logs to ./all_checkpoints/stage1_06282348_ddp/wandb/run-20250628_235448-d21a8n96/logs/debug-internal.log
+2025-06-28 23:54:48,944 INFO    MainThread:908035 [wandb_init.py:init():852] calling init triggers
+2025-06-28 23:54:48,944 INFO    MainThread:908035 [wandb_init.py:init():857] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-06-28 23:54:48,944 INFO    MainThread:908035 [wandb_init.py:init():893] starting backend
+2025-06-28 23:54:48,944 INFO    MainThread:908035 [wandb_init.py:init():897] sending inform_init request
+2025-06-28 23:54:48,946 INFO    MainThread:908035 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-06-28 23:54:48,948 INFO    MainThread:908035 [wandb_init.py:init():907] backend started and connected
+2025-06-28 23:54:48,952 INFO    MainThread:908035 [wandb_init.py:init():1005] updated telemetry
+2025-06-28 23:54:48,955 INFO    MainThread:908035 [wandb_init.py:init():1029] communicating run to backend with 90.0 second timeout
+2025-06-28 23:55:15,683 INFO    MainThread:908035 [wandb_init.py:init():1104] starting run threads in backend
+2025-06-28 23:55:15,875 INFO    MainThread:908035 [wandb_run.py:_console_start():2573] atexit reg
+2025-06-28 23:55:15,875 INFO    MainThread:908035 [wandb_run.py:_redirect():2421] redirect: wrap_raw
+2025-06-28 23:55:15,879 INFO    MainThread:908035 [wandb_run.py:_redirect():2490] Wrapping output streams.
+2025-06-28 23:55:15,879 INFO    MainThread:908035 [wandb_run.py:_redirect():2513] Redirects installed.
+2025-06-28 23:55:15,880 INFO    MainThread:908035 [wandb_init.py:init():1150] run started, returning control to user process
+2025-06-29 00:01:37,881 INFO    MsgRouterThr:908035 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles.

ProtT3/all_checkpoints/stage1_06282348_ddp/wandb/run-20250629_000421-y2lylvs5/logs/debug-internal.log

@@ -0,0 +1,16 @@
+{"time":"2025-06-29T00:04:21.83840344+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.11","symlink path":"all_checkpoints/stage1_06282348_ddp/wandb/run-20250629_000421-y2lylvs5/logs/debug-core.log"}
+{"time":"2025-06-29T00:04:46.244498134+08:00","level":"INFO","msg":"created new stream","id":"y2lylvs5"}
+{"time":"2025-06-29T00:04:46.245490243+08:00","level":"INFO","msg":"stream: started","id":"y2lylvs5"}
+{"time":"2025-06-29T00:04:46.245519608+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"y2lylvs5"}
+{"time":"2025-06-29T00:04:46.245575002+08:00","level":"INFO","msg":"handler: started","stream_id":"y2lylvs5"}
+{"time":"2025-06-29T00:04:46.245579305+08:00","level":"INFO","msg":"sender: started","stream_id":"y2lylvs5"}
+{"time":"2025-06-29T00:05:16.250479528+08:00","level":"ERROR","msg":"send: sendRun: failed to update run state: context deadline exceeded (Client.Timeout or context cancellation while reading body)"}
+{"time":"2025-06-29T00:05:17.262869937+08:00","level":"INFO","msg":"stream: closing","id":"y2lylvs5"}
+{"time":"2025-06-29T00:05:17.262985061+08:00","level":"ERROR","msg":"sender: upsertConfig: RunRecord is nil"}
+{"time":"2025-06-29T00:05:22.061017174+08:00","level":"ERROR","msg":"HTTP error","status":404,"method":"POST","url":"https://api.bandw.top/graphql"}
+{"time":"2025-06-29T00:05:22.061118361+08:00","level":"ERROR","msg":"runfiles: CreateRunFiles returned error: returned error 404: {\"data\":{\"createRunFiles\":null},\"errors\":[{\"message\":\"project /stage1_06282348_ddp not found during createRunFiles\",\"path\":[\"createRunFiles\"]}]}"}
+{"time":"2025-06-29T00:05:22.066030873+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-06-29T00:05:22.067356057+08:00","level":"INFO","msg":"handler: closed","stream_id":"y2lylvs5"}
+{"time":"2025-06-29T00:05:22.067390632+08:00","level":"INFO","msg":"sender: closed","stream_id":"y2lylvs5"}
+{"time":"2025-06-29T00:05:22.067390315+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"y2lylvs5"}
+{"time":"2025-06-29T00:05:22.070426682+08:00","level":"INFO","msg":"stream: closed","id":"y2lylvs5"}

ProtT3/all_checkpoints/stage1_06282348_ddp/wandb/run-20250629_000421-y2lylvs5/logs/debug.log

@@ -0,0 +1,16 @@
+2025-06-29 00:04:21,827 INFO    MainThread:937487 [wandb_setup.py:_flush():70] Current SDK version is 0.19.11
+2025-06-29 00:04:21,827 INFO    MainThread:937487 [wandb_setup.py:_flush():70] Configure stats pid to 937487
+2025-06-29 00:04:21,827 INFO    MainThread:937487 [wandb_setup.py:_flush():70] Loading settings from /root/.config/wandb/settings
+2025-06-29 00:04:21,827 INFO    MainThread:937487 [wandb_setup.py:_flush():70] Loading settings from /nas/shared/kilab/wangyujia/ProtT3/wandb/settings
+2025-06-29 00:04:21,827 INFO    MainThread:937487 [wandb_setup.py:_flush():70] Loading settings from environment variables
+2025-06-29 00:04:21,827 INFO    MainThread:937487 [wandb_init.py:setup_run_log_directory():724] Logging user logs to ./all_checkpoints/stage1_06282348_ddp/wandb/run-20250629_000421-y2lylvs5/logs/debug.log
+2025-06-29 00:04:21,827 INFO    MainThread:937487 [wandb_init.py:setup_run_log_directory():725] Logging internal logs to ./all_checkpoints/stage1_06282348_ddp/wandb/run-20250629_000421-y2lylvs5/logs/debug-internal.log
+2025-06-29 00:04:21,827 INFO    MainThread:937487 [wandb_init.py:init():852] calling init triggers
+2025-06-29 00:04:21,828 INFO    MainThread:937487 [wandb_init.py:init():857] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-06-29 00:04:21,828 INFO    MainThread:937487 [wandb_init.py:init():893] starting backend
+2025-06-29 00:04:21,828 INFO    MainThread:937487 [wandb_init.py:init():897] sending inform_init request
+2025-06-29 00:04:21,829 INFO    MainThread:937487 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-06-29 00:04:21,832 INFO    MainThread:937487 [wandb_init.py:init():907] backend started and connected
+2025-06-29 00:04:21,838 INFO    MainThread:937487 [wandb_init.py:init():1005] updated telemetry
+2025-06-29 00:04:21,842 INFO    MainThread:937487 [wandb_init.py:init():1029] communicating run to backend with 90.0 second timeout

ProtT3/all_checkpoints/stage1_06282348_ddp/wandb/run-20250629_000702-rypk39yq/files/config.yaml

@@ -0,0 +1,237 @@
+_wandb:
+    value:
+        cli_version: 0.19.11
+        m:
+            - "1": lr
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": trainer/global_step
+              "6":
+                - 3
+              "7": []
+            - "1": loader0/val_loss_ptm/dataloader_idx_0
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loader2/val_loss/dataloader_idx_2
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loader2/val_loss_lm/dataloader_idx_2
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": epoch
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loader2/val_loss_ptc/dataloader_idx_2
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loader0/val_loss_ptc/dataloader_idx_0
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": train_loss_lm
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": train_loss
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loader1/val_loss_ptc/dataloader_idx_1
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loader1/val_loss_ptm/dataloader_idx_1
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loader0/val_loss/dataloader_idx_0
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loader0/val_loss_lm/dataloader_idx_0
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": train_loss_ptm
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": train_loss_ptc
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loader1/val_loss_lm/dataloader_idx_1
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loader1/val_loss/dataloader_idx_1
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loader2/val_loss_ptm/dataloader_idx_2
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+        python_version: 3.10.0
+        t:
+            "1":
+                - 1
+                - 5
+                - 9
+                - 11
+                - 33
+                - 41
+                - 49
+                - 53
+                - 55
+                - 63
+                - 103
+            "2":
+                - 1
+                - 5
+                - 9
+                - 11
+                - 33
+                - 41
+                - 49
+                - 53
+                - 55
+                - 63
+                - 103
+            "3":
+                - 7
+                - 23
+                - 33
+                - 55
+                - 66
+            "4": 3.10.0
+            "5": 0.19.11
+            "6": 4.52.3
+            "8":
+                - 5
+            "12": 0.19.11
+            "13": linux-x86_64
+accelerator:
+    value: gpu
+batch_size:
+    value: 96
+bert_hidden_dim:
+    value: 768
+bert_name:
+    value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft
+check_val_every_n_epoch:
+    value: 1
+cross_attention_freq:
+    value: 2
+devices:
+    value: 0,1,2,3,4,5,6,7
+filename:
+    value: stage1_06282348_ddp
+init_checkpoint:
+    value: ""
+init_lr:
+    value: 0.0001
+lm:
+    value: true
+load_4bit:
+    value: false
+lr_decay_rate:
+    value: 0.9
+match_batch_size:
+    value: 64
+max_epochs:
+    value: 20
+min_lr:
+    value: 1e-05
+mix_dataset:
+    value: true
+mode:
+    value: train
+num_query_token:
+    value: 8
+num_workers:
+    value: 8
+plm_name:
+    value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m
+plm_tune:
+    value: freeze
+pool_size:
+    value: 0
+precision:
+    value: bf16-mixed
+projection_dim:
+    value: 256
+prot_aug:
+    value: None
+prot_max_len:
+    value: 1024
+ptm:
+    value: true
+rerank_cand_num:
+    value: 128
+retrieval_eval_epoch:
+    value: 10
+root:
+    value: data
+save_every_n_epochs:
+    value: 5
+scheduler:
+    value: linear_warmup_cosine_lr
+seed:
+    value: 42
+strategy:
+    value: ddp
+temperature:
+    value: 0.1
+text_max_len:
+    value: 128
+use_wandb_logger:
+    value: true
+warmup_lr:
+    value: 1e-06
+warmup_steps:
+    value: 1000
+weight_decay:
+    value: 0.05

ProtT3/all_checkpoints/stage1_06282348_ddp/wandb/run-20250629_000702-rypk39yq/files/requirements.txt

@@ -0,0 +1,225 @@
+stack-data==0.6.3
+yarl==1.20.0
+setuptools==78.1.1
+cloudpathlib==0.21.1
+pytz==2025.2
+nvidia-cufile-cu12==1.11.1.6
+shellingham==1.5.4
+nvidia-cusolver-cu12==11.6.1.9
+Jinja2==3.1.6
+pycocotools==2.0.8
+pandas==2.2.3
+scipy==1.15.3
+tenacity==9.1.2
+lightning-utilities==0.14.3
+cfgv==3.4.0
+hf-xet==1.1.2
+platformdirs==4.3.8
+smart-open==7.1.0
+text-unidecode==1.3
+nvidia-cublas-cu12==12.4.5.8
+omegaconf==2.3.0
+referencing==0.36.2
+mdurl==0.1.2
+gitdb==4.0.12
+identify==2.6.12
+ipython==8.36.0
+spacy-loggers==1.0.5
+distlib==0.3.9
+typing-inspection==0.4.1
+antlr4-python3-runtime==4.9.3
+multidict==6.4.4
+nvidia-curand-cu12==10.3.5.147
+prompt_toolkit==3.0.51
+Pygments==2.19.1
+numpy==2.2.6
+decord==0.6.0
+srsly==2.5.1
+watchdog==6.0.0
+pure_eval==0.2.3
+virtualenv==20.31.2
+altair==5.5.0
+matplotlib-inline==0.1.7
+bleach==6.2.0
+exceptiongroup==1.3.0
+fairscale==0.4.4
+confection==0.1.5
+fonttools==4.58.0
+nvidia-cuda-nvrtc-cu12==12.4.127
+ptyprocess==0.7.0
+pytorch-lightning==2.5.1.post0
+nodeenv==1.9.1
+nvidia-cudnn-cu12==9.1.0.70
+requests==2.32.3
+marisa-trie==1.2.1
+cachetools==5.5.2
+matplotlib==3.10.3
+typing_extensions==4.13.2
+asttokens==3.0.0
+torch==2.6.0
+PyYAML==6.0.2
+tifffile==2025.5.10
+spacy==3.8.7
+braceexpand==0.1.7
+plotly==6.1.1
+attrs==25.3.0
+py-cpuinfo==9.0.0
+frozenlist==1.6.0
+catalogue==2.0.10
+nvidia-cusparselt-cu12==0.6.2
+traitlets==5.14.3
+annotated-types==0.7.0
+language_data==1.3.0
+thinc==8.3.6
+imageio==2.37.0
+nvidia-cuda-runtime-cu12==12.4.127
+certifi==2025.4.26
+smmap==5.0.2
+python-magic==0.4.27
+triton==3.2.0
+weasel==0.4.1
+async-timeout==5.0.1
+wcwidth==0.2.13
+pillow==11.2.1
+torchmetrics==1.7.1
+kaggle==1.7.4.5
+regex==2024.11.6
+aiosignal==1.3.2
+nvidia-cusparse-cu12==12.3.1.170
+scikit-image==0.25.2
+nvidia-nvtx-cu12==12.4.127
+opendatasets==0.1.22
+iopath==0.1.10
+pyparsing==3.2.3
+portalocker==3.1.1
+executing==2.2.0
+contexttimer==0.3.3
+lazy_loader==0.4
+wrapt==1.17.2
+webdataset==0.2.111
+blis==1.3.0
+idna==3.10
+timm==0.4.12
+einops==0.8.1
+packaging==24.2
+decorator==5.2.1
+filelock==3.18.0
+python-slugify==8.0.4
+cycler==0.12.1
+charset-normalizer==3.4.2
+pydantic==2.11.5
+pydeck==0.9.1
+tzdata==2025.2
+jedi==0.19.2
+aiohappyeyeballs==2.6.1
+nvidia-nvjitlink-cu12==12.4.127
+salesforce-lavis==1.0.2
+parso==0.8.4
+nvidia-nccl-cu12==2.21.5
+toml==0.10.2
+python-dateutil==2.9.0.post0
+rich==14.0.0
+tqdm==4.67.1
+rpds-py==0.25.1
+opencv-python-headless==4.5.5.64
+tornado==6.5.1
+propcache==0.3.1
+webencodings==0.5.1
+murmurhash==1.0.13
+contourpy==1.3.2
+joblib==1.5.1
+networkx==3.4.2
+six==1.17.0
+markdown-it-py==3.0.0
+nvidia-cuda-cupti-cu12==12.4.127
+msgpack==1.1.0
+sentencepiece==0.2.0
+cymem==2.0.11
+nvidia-cufft-cu12==11.2.1.3
+absl-py==2.2.2
+hjson==3.1.0
+mpmath==1.3.0
+pydantic_core==2.33.2
+psutil==7.0.0
+nvidia-ml-py==12.575.51
+pyarrow==20.0.0
+kiwisolver==1.4.8
+sympy==1.13.1
+ninja==1.11.1.4
+rouge_score==0.1.2
+deepspeed==0.16.10+b666844f
+spacy-legacy==3.0.12
+pycocoevalcap==1.2
+pexpect==4.9.0
+ftfy==6.3.1
+protobuf==6.31.0
+urllib3==2.4.0
+wheel==0.45.1
+nltk==3.9.1
+streamlit==1.45.1
+wasabi==1.1.3
+pre_commit==4.2.0
+safetensors==0.5.3
+jsonschema-specifications==2025.4.1
+langcodes==3.5.0
+GitPython==3.1.44
+blinker==1.9.0
+torchvision==0.21.0
+MarkupSafe==3.0.2
+dill==0.3.8
+yacs==0.1.8
+pathlib==1.0.1
+scikit-learn==1.6.1
+cffi==1.17.1
+pycparser==2.22
+flash-attn==2.7.1.post1
+cryptography==45.0.3
+pycryptodome==3.23.0
+cheroot==10.0.1
+more-itertools==10.7.0
+setproctitle==1.3.6
+delta-center-client==0.0.4
+jmespath==0.10.0
+xxhash==3.5.0
+pip==25.1.1
+aliyun-python-sdk-core==2.16.0
+jaraco.functools==4.1.0
+bigmodelvis==0.0.1
+aiohttp==3.12.2
+multiprocess==0.70.16
+opendelta==0.3.2
+docker-pycreds==0.4.0
+threadpoolctl==3.6.0
+click==8.2.1
+oss2==2.15.0
+crcmod==1.7
+transformers==4.52.3
+datasets==3.6.0
+jsonschema==4.24.0
+opencv-python==4.11.0.86
+wandb==0.19.11
+fsspec==2025.3.0
+tokenizers==0.21.1
+sentry-sdk==2.29.1
+preshed==3.0.10
+aliyun-python-sdk-kms==2.16.5
+huggingface-hub==0.32.1
+typer==0.16.0
+narwhals==1.41.0
+web.py==0.62
+autocommand==2.2.2
+importlib_metadata==8.0.0
+zipp==3.19.2
+jaraco.context==5.3.0
+typeguard==4.3.0
+jaraco.collections==5.1.0
+typing_extensions==4.12.2
+backports.tarfile==1.2.0
+jaraco.functools==4.0.1
+more-itertools==10.3.0
+platformdirs==4.2.2
+packaging==24.2
+tomli==2.0.1
+jaraco.text==3.12.1
+wheel==0.45.1
+inflect==7.3.1

ProtT3/all_checkpoints/stage1_06282348_ddp/wandb/run-20250629_000702-rypk39yq/files/wandb-metadata.json

@@ -0,0 +1,99 @@
+{
+  "os": "Linux-5.10.134-008.16.kangaroo.al8.x86_64-x86_64-with-glibc2.35",
+  "python": "CPython 3.10.0",
+  "startedAt": "2025-06-28T16:07:02.125247Z",
+  "args": [
+    "--devices",
+    "0,1,2,3,4,5,6,7",
+    "--mode",
+    "train",
+    "--filename",
+    "stage1_06282348_ddp",
+    "--num_query_token",
+    "8",
+    "--plm_name",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m",
+    "--bert_name",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft",
+    "--save_every_n_epochs",
+    "5",
+    "--max_epochs",
+    "20",
+    "--batch_size",
+    "96",
+    "--precision",
+    "bf16-mixed",
+    "--mix_dataset",
+    "--num_workers",
+    "8",
+    "--use_wandb_logger",
+    "--strategy",
+    "ddp"
+  ],
+  "program": "/nas/shared/kilab/wangyujia/ProtT3/stage1.py",
+  "codePath": "stage1.py",
+  "root": "./all_checkpoints/stage1_06282348_ddp/",
+  "host": "dsw-265304-57b7b77cbc-vwbwc",
+  "executable": "/root/miniconda3/envs/protT3/bin/python",
+  "codePathLocal": "stage1.py",
+  "cpu_count": 64,
+  "cpu_count_logical": 64,
+  "gpu": "NVIDIA A800-SXM4-80GB",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1623302262784",
+      "used": "1285242880"
+    }
+  },
+  "memory": {
+    "total": "549755813888"
+  },
+  "cpu": {
+    "count": 64,
+    "countLogical": 64
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    }
+  ],
+  "cudaVersion": "12.1"
+}

ProtT3/all_checkpoints/stage1_06282348_ddp/wandb/run-20250629_000702-rypk39yq/files/wandb-summary.json

@@ -0,0 +1 @@
+{"loader1/val_loss_lm/dataloader_idx_1":6.895899772644043,"loader2/val_loss_lm/dataloader_idx_2":7.282508373260498,"loader0/val_loss_ptc/dataloader_idx_0":0.645770788192749,"loader0/val_loss_lm/dataloader_idx_0":2.0093350410461426,"loader2/val_loss_ptc/dataloader_idx_2":4.874267101287842,"_step":243,"lr":1.0554024811426643e-05,"loader0/val_loss/dataloader_idx_0":3.0574357509613037,"train_loss_lm":1.5327174663543701,"train_loss":2.090651035308838,"loader0/val_loss_ptm/dataloader_idx_0":0.4023294746875763,"_runtime":36514.777432942,"loader2/val_loss_ptm/dataloader_idx_2":2.5906364917755127,"trainer/global_step":11199,"loader1/val_loss/dataloader_idx_1":14.83725357055664,"loader1/val_loss_ptm/dataloader_idx_1":2.7793824672698975,"_wandb":{"runtime":36546},"loader2/val_loss/dataloader_idx_2":14.74741268157959,"train_loss_ptm":0.22319991886615753,"train_loss_ptc":0.33473363518714905,"_timestamp":1.751163336902292e+09,"loader1/val_loss_ptc/dataloader_idx_1":5.1619720458984375,"epoch":19}

ProtT3/all_checkpoints/stage1_06290009_deepspeed/wandb/debug-internal.log

@@ -0,0 +1,42 @@
+{"time":"2025-06-29T00:12:00.930911849+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.11","symlink path":"all_checkpoints/stage1_06290009_deepspeed/wandb/run-20250629_001200-vgvxxzqc/logs/debug-core.log"}
+{"time":"2025-06-29T00:12:31.03707486+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:12:36.876219526+08:00","level":"INFO","msg":"created new stream","id":"vgvxxzqc"}
+{"time":"2025-06-29T00:12:36.876272436+08:00","level":"INFO","msg":"stream: started","id":"vgvxxzqc"}
+{"time":"2025-06-29T00:12:36.876317878+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"vgvxxzqc"}
+{"time":"2025-06-29T00:12:36.876360145+08:00","level":"INFO","msg":"handler: started","stream_id":"vgvxxzqc"}
+{"time":"2025-06-29T00:12:36.876401621+08:00","level":"INFO","msg":"sender: started","stream_id":"vgvxxzqc"}
+{"time":"2025-06-29T00:12:39.839397838+08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2025-06-29T00:13:20.87652364+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:58014->104.21.20.172:443: read: connection reset by peer"}
+{"time":"2025-06-29T00:16:17.426969211+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": unexpected EOF"}
+{"time":"2025-06-29T00:17:58.51721346+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": unexpected EOF"}
+{"time":"2025-06-29T00:26:10.222503445+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:51478->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-06-29T00:28:34.807681677+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:46066->104.21.20.172:443: read: connection reset by peer"}
+{"time":"2025-06-29T00:33:01.870533298+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:55734->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-29T00:33:23.754842161+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:46042->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-06-29T00:36:47.149592254+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:57176->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-06-29T00:41:05.645712211+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": unexpected EOF"}
+{"time":"2025-06-29T00:47:20.493525378+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:58300->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-29T00:47:45.629981285+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": unexpected EOF"}
+{"time":"2025-06-29T00:51:19.597480154+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:60802->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-29T00:54:09.777365701+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:51532->104.21.20.172:443: read: connection reset by peer"}
+{"time":"2025-06-29T01:00:20.78154967+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:34934->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-06-29T01:04:21.421531776+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:55938->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-29T01:05:41.05509194+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:45678->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-06-29T01:08:44.130266043+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": unexpected EOF"}
+{"time":"2025-06-29T01:10:59.724692621+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": unexpected EOF"}
+{"time":"2025-06-29T01:14:35.821570745+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:35232->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-29T01:17:35.533530754+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:44258->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-29T01:23:12.630381225+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:48108->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-06-29T01:24:55.067569821+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T01:25:27.188065501+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T01:25:56.782489942+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:45024->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-29T01:26:49.538892815+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": unexpected EOF"}
+{"time":"2025-06-29T01:29:46.157546022+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:42508->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-06-29T05:53:57.9412544+08:00","level":"INFO","msg":"stream: closing","id":"vgvxxzqc"}
+{"time":"2025-06-29T05:53:57.941286983+08:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2025-06-29T05:53:57.942366437+08:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2025-06-29T05:54:00.869660002+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-06-29T05:54:03.731237694+08:00","level":"INFO","msg":"handler: closed","stream_id":"vgvxxzqc"}
+{"time":"2025-06-29T05:54:03.731282348+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"vgvxxzqc"}
+{"time":"2025-06-29T05:54:03.731313818+08:00","level":"INFO","msg":"sender: closed","stream_id":"vgvxxzqc"}
+{"time":"2025-06-29T05:54:03.735031072+08:00","level":"INFO","msg":"stream: closed","id":"vgvxxzqc"}

ProtT3/all_checkpoints/stage1_06290009_deepspeed/wandb/debug.log

@@ -0,0 +1,24 @@
+2025-06-29 00:12:00,920 INFO    MainThread:2351 [wandb_setup.py:_flush():70] Current SDK version is 0.19.11
+2025-06-29 00:12:00,920 INFO    MainThread:2351 [wandb_setup.py:_flush():70] Configure stats pid to 2351
+2025-06-29 00:12:00,920 INFO    MainThread:2351 [wandb_setup.py:_flush():70] Loading settings from /root/.config/wandb/settings
+2025-06-29 00:12:00,920 INFO    MainThread:2351 [wandb_setup.py:_flush():70] Loading settings from /nas/shared/kilab/wangyujia/ProtT3/wandb/settings
+2025-06-29 00:12:00,920 INFO    MainThread:2351 [wandb_setup.py:_flush():70] Loading settings from environment variables
+2025-06-29 00:12:00,920 INFO    MainThread:2351 [wandb_init.py:setup_run_log_directory():724] Logging user logs to ./all_checkpoints/stage1_06290009_deepspeed/wandb/run-20250629_001200-vgvxxzqc/logs/debug.log
+2025-06-29 00:12:00,920 INFO    MainThread:2351 [wandb_init.py:setup_run_log_directory():725] Logging internal logs to ./all_checkpoints/stage1_06290009_deepspeed/wandb/run-20250629_001200-vgvxxzqc/logs/debug-internal.log
+2025-06-29 00:12:00,920 INFO    MainThread:2351 [wandb_init.py:init():852] calling init triggers
+2025-06-29 00:12:00,921 INFO    MainThread:2351 [wandb_init.py:init():857] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-06-29 00:12:00,921 INFO    MainThread:2351 [wandb_init.py:init():893] starting backend
+2025-06-29 00:12:00,921 INFO    MainThread:2351 [wandb_init.py:init():897] sending inform_init request
+2025-06-29 00:12:00,923 INFO    MainThread:2351 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-06-29 00:12:00,924 INFO    MainThread:2351 [wandb_init.py:init():907] backend started and connected
+2025-06-29 00:12:00,926 INFO    MainThread:2351 [wandb_init.py:init():1005] updated telemetry
+2025-06-29 00:12:00,929 INFO    MainThread:2351 [wandb_init.py:init():1029] communicating run to backend with 90.0 second timeout
+2025-06-29 00:12:39,788 INFO    MainThread:2351 [wandb_init.py:init():1104] starting run threads in backend
+2025-06-29 00:12:40,030 INFO    MainThread:2351 [wandb_run.py:_console_start():2573] atexit reg
+2025-06-29 00:12:40,030 INFO    MainThread:2351 [wandb_run.py:_redirect():2421] redirect: wrap_raw
+2025-06-29 00:12:40,034 INFO    MainThread:2351 [wandb_run.py:_redirect():2490] Wrapping output streams.
+2025-06-29 00:12:40,034 INFO    MainThread:2351 [wandb_run.py:_redirect():2513] Redirects installed.
+2025-06-29 00:12:40,036 INFO    MainThread:2351 [wandb_init.py:init():1150] run started, returning control to user process
+2025-06-29 00:12:46,669 INFO    MainThread:2351 [wandb_run.py:_config_callback():1436] config_cb None None {'filename': 'stage1_06290009_deepspeed', 'seed': 42, 'mode': 'train', 'strategy': 'deepspeed', 'accelerator': 'gpu', 'devices': '0,1,2,3,4,5,6,7', 'precision': 'bf16-mixed', 'max_epochs': 20, 'check_val_every_n_epoch': 1, 'use_wandb_logger': True, 'mix_dataset': True, 'temperature': 0.1, 'save_every_n_epochs': 5, 'ptm': True, 'lm': True, 'rerank_cand_num': 128, 'plm_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m', 'plm_tune': 'freeze', 'load_4bit': False, 'pool_size': 0, 'bert_hidden_dim': 768, 'bert_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft', 'projection_dim': 256, 'cross_attention_freq': 2, 'num_query_token': 8, 'weight_decay': 0.05, 'init_lr': 0.0001, 'min_lr': 1e-05, 'warmup_lr': 1e-06, 'warmup_steps': 1000, 'lr_decay_rate': 0.9, 'scheduler': 'linear_warmup_cosine_lr', 'init_checkpoint': '', 'retrieval_eval_epoch': 10, 'num_workers': 8, 'batch_size': 168, 'match_batch_size': 64, 'root': 'data', 'text_max_len': 128, 'prot_max_len': 1024, 'prot_aug': 'None'}
+2025-06-29 05:53:57,929 INFO    MsgRouterThr:2351 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 2 handles.

ProtT3/all_checkpoints/stage1_06290009_deepspeed/wandb/run-20250629_001200-vgvxxzqc/files/config.yaml

@@ -0,0 +1,236 @@
+_wandb:
+    value:
+        cli_version: 0.19.11
+        m:
+            - "1": trainer/global_step
+              "6":
+                - 3
+              "7": []
+            - "1": train_loss
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loader1/val_loss/dataloader_idx_1
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loader0/val_loss_lm/dataloader_idx_0
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loader1/val_loss_ptm/dataloader_idx_1
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loader2/val_loss_lm/dataloader_idx_2
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loader1/val_loss_ptc/dataloader_idx_1
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": epoch
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": lr
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loader2/val_loss_ptc/dataloader_idx_2
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loader0/val_loss_ptm/dataloader_idx_0
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": train_loss_ptc
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": train_loss_ptm
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": train_loss_lm
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loader2/val_loss_ptm/dataloader_idx_2
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loader0/val_loss/dataloader_idx_0
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loader2/val_loss/dataloader_idx_2
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loader1/val_loss_lm/dataloader_idx_1
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loader0/val_loss_ptc/dataloader_idx_0
+              "5": 1
+              "6":
+                - 1
+                - 3
+              "7": []
+        python_version: 3.10.0
+        t:
+            "1":
+                - 1
+                - 5
+                - 9
+                - 11
+                - 33
+                - 41
+                - 49
+                - 53
+                - 55
+                - 63
+                - 103
+            "2":
+                - 1
+                - 5
+                - 9
+                - 11
+                - 33
+                - 41
+                - 49
+                - 53
+                - 55
+                - 63
+                - 103
+            "3":
+                - 7
+                - 23
+                - 55
+                - 66
+            "4": 3.10.0
+            "5": 0.19.11
+            "6": 4.52.3
+            "8":
+                - 5
+            "12": 0.19.11
+            "13": linux-x86_64
+accelerator:
+    value: gpu
+batch_size:
+    value: 168
+bert_hidden_dim:
+    value: 768
+bert_name:
+    value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft
+check_val_every_n_epoch:
+    value: 1
+cross_attention_freq:
+    value: 2
+devices:
+    value: 0,1,2,3,4,5,6,7
+filename:
+    value: stage1_06290009_deepspeed
+init_checkpoint:
+    value: ""
+init_lr:
+    value: 0.0001
+lm:
+    value: true
+load_4bit:
+    value: false
+lr_decay_rate:
+    value: 0.9
+match_batch_size:
+    value: 64
+max_epochs:
+    value: 20
+min_lr:
+    value: 1e-05
+mix_dataset:
+    value: true
+mode:
+    value: train
+num_query_token:
+    value: 8
+num_workers:
+    value: 8
+plm_name:
+    value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m
+plm_tune:
+    value: freeze
+pool_size:
+    value: 0
+precision:
+    value: bf16-mixed
+projection_dim:
+    value: 256
+prot_aug:
+    value: None
+prot_max_len:
+    value: 1024
+ptm:
+    value: true
+rerank_cand_num:
+    value: 128
+retrieval_eval_epoch:
+    value: 10
+root:
+    value: data
+save_every_n_epochs:
+    value: 5
+scheduler:
+    value: linear_warmup_cosine_lr
+seed:
+    value: 42
+strategy:
+    value: deepspeed
+temperature:
+    value: 0.1
+text_max_len:
+    value: 128
+use_wandb_logger:
+    value: true
+warmup_lr:
+    value: 1e-06
+warmup_steps:
+    value: 1000
+weight_decay:
+    value: 0.05

ProtT3/all_checkpoints/stage1_06290009_deepspeed/wandb/run-20250629_001200-vgvxxzqc/files/output.log

@@ -0,0 +1,21 @@
+/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:654: Checkpoint directory /nas/shared/kilab/wangyujia/ProtT3/all_checkpoints/stage1_06290009_deepspeed exists and is not empty.
+Enabling DeepSpeed BF16. Model parameters and inputs will be cast to `bfloat16`.
+LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]
+
+  | Name         | Type         | Params | Mode
+------------------------------------------------------
+0 | blip2qformer | Blip2Qformer | 327 M  | train
+------------------------------------------------------
+179 M     Trainable params
+147 M     Non-trainable params
+327 M     Total params
+1,309.467 Total estimated model params size (MB)
+5         Modules in train mode
+926       Modules in eval mode
+Epoch 19: 100%|███████████████████████████████████████████| 320/320 [17:07<00:00,  0.31it/s, v_num=xzqc]
+/nas/shared/kilab/wangyujia/ProtT3/model/blip2qformer.py:220: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(enable_autocast, dtype=torch.float32):
+                                                                                                        
+/nas/shared/kilab/wangyujia/ProtT3/model/dist_funs.py:18: FutureWarning: Positional args are being deprecated, use kwargs instead. Refer to https://pytorch.org/docs/main/generated/torch.nn.Module.html#torch.nn.Module.state_dict for details.
+  sd = self.module.state_dict(destination, prefix, keep_vars)
+`Trainer.fit` stopped: `max_epochs=20` reached.

ProtT3/all_checkpoints/stage1_06290009_deepspeed/wandb/run-20250629_001200-vgvxxzqc/files/requirements.txt

@@ -0,0 +1,225 @@
+opendatasets==0.1.22
+salesforce-lavis==1.0.2
+Pygments==2.19.1
+nvidia-nccl-cu12==2.21.5
+tornado==6.5.1
+nvidia-cuda-runtime-cu12==12.4.127
+requests==2.32.3
+nvidia-cuda-cupti-cu12==12.4.127
+decord==0.6.0
+braceexpand==0.1.7
+frozenlist==1.6.0
+markdown-it-py==3.0.0
+shellingham==1.5.4
+absl-py==2.2.2
+pycocoevalcap==1.2
+contexttimer==0.3.3
+bleach==6.2.0
+jsonschema-specifications==2025.4.1
+pycocotools==2.0.8
+python-slugify==8.0.4
+tqdm==4.67.1
+numpy==2.2.6
+urllib3==2.4.0
+deepspeed==0.16.10+b666844f
+watchdog==6.0.0
+wrapt==1.17.2
+setuptools==78.1.1
+matplotlib==3.10.3
+pydeck==0.9.1
+aiosignal==1.3.2
+gitdb==4.0.12
+hjson==3.1.0
+timm==0.4.12
+blis==1.3.0
+PyYAML==6.0.2
+referencing==0.36.2
+contourpy==1.3.2
+kaggle==1.7.4.5
+triton==3.2.0
+catalogue==2.0.10
+idna==3.10
+torch==2.6.0
+text-unidecode==1.3
+altair==5.5.0
+cloudpathlib==0.21.1
+protobuf==6.31.0
+nvidia-cusolver-cu12==11.6.1.9
+pytz==2025.2
+sympy==1.13.1
+spacy==3.8.7
+MarkupSafe==3.0.2
+thinc==8.3.6
+nvidia-cudnn-cu12==9.1.0.70
+wasabi==1.1.3
+aiohappyeyeballs==2.6.1
+nvidia-nvtx-cu12==12.4.127
+rich==14.0.0
+ipython==8.36.0
+yarl==1.20.0
+torchmetrics==1.7.1
+multidict==6.4.4
+cfgv==3.4.0
+smmap==5.0.2
+srsly==2.5.1
+scikit-image==0.25.2
+matplotlib-inline==0.1.7
+annotated-types==0.7.0
+lazy_loader==0.4
+tenacity==9.1.2
+GitPython==3.1.44
+language_data==1.3.0
+pydantic_core==2.33.2
+sentencepiece==0.2.0
+platformdirs==4.3.8
+distlib==0.3.9
+nvidia-cusparselt-cu12==0.6.2
+blinker==1.9.0
+regex==2024.11.6
+tifffile==2025.5.10
+py-cpuinfo==9.0.0
+attrs==25.3.0
+mdurl==0.1.2
+prompt_toolkit==3.0.51
+packaging==24.2
+async-timeout==5.0.1
+six==1.17.0
+executing==2.2.0
+parso==0.8.4
+omegaconf==2.3.0
+wcwidth==0.2.13
+murmurhash==1.0.13
+stack-data==0.6.3
+nvidia-cufft-cu12==11.2.1.3
+virtualenv==20.31.2
+langcodes==3.5.0
+fonttools==4.58.0
+opencv-python-headless==4.5.5.64
+jedi==0.19.2
+torchvision==0.21.0
+plotly==6.1.1
+nodeenv==1.9.1
+smart-open==7.1.0
+toml==0.10.2
+pytorch-lightning==2.5.1.post0
+typing_extensions==4.13.2
+safetensors==0.5.3
+psutil==7.0.0
+pillow==11.2.1
+python-dateutil==2.9.0.post0
+ftfy==6.3.1
+scipy==1.15.3
+webdataset==0.2.111
+charset-normalizer==3.4.2
+nvidia-nvjitlink-cu12==12.4.127
+kiwisolver==1.4.8
+nvidia-ml-py==12.575.51
+confection==0.1.5
+nvidia-curand-cu12==10.3.5.147
+pandas==2.2.3
+nltk==3.9.1
+webencodings==0.5.1
+pyarrow==20.0.0
+asttokens==3.0.0
+exceptiongroup==1.3.0
+pre_commit==4.2.0
+ninja==1.11.1.4
+spacy-loggers==1.0.5
+msgpack==1.1.0
+lightning-utilities==0.14.3
+nvidia-cublas-cu12==12.4.5.8
+tzdata==2025.2
+cycler==0.12.1
+hf-xet==1.1.2
+antlr4-python3-runtime==4.9.3
+iopath==0.1.10
+pexpect==4.9.0
+imageio==2.37.0
+streamlit==1.45.1
+python-magic==0.4.27
+networkx==3.4.2
+portalocker==3.1.1
+nvidia-cusparse-cu12==12.3.1.170
+propcache==0.3.1
+ptyprocess==0.7.0
+fairscale==0.4.4
+rpds-py==0.25.1
+certifi==2025.4.26
+rouge_score==0.1.2
+traitlets==5.14.3
+identify==2.6.12
+spacy-legacy==3.0.12
+weasel==0.4.1
+mpmath==1.3.0
+cymem==2.0.11
+typing-inspection==0.4.1
+nvidia-cuda-nvrtc-cu12==12.4.127
+marisa-trie==1.2.1
+einops==0.8.1
+nvidia-cufile-cu12==1.11.1.6
+pydantic==2.11.5
+cachetools==5.5.2
+joblib==1.5.1
+Jinja2==3.1.6
+filelock==3.18.0
+pyparsing==3.2.3
+pure_eval==0.2.3
+decorator==5.2.1
+wheel==0.45.1
+pycryptodome==3.23.0
+cheroot==10.0.1
+multiprocess==0.70.16
+aiohttp==3.12.2
+crcmod==1.7
+fsspec==2025.3.0
+jmespath==0.10.0
+preshed==3.0.10
+jaraco.functools==4.1.0
+cryptography==45.0.3
+sentry-sdk==2.29.1
+tokenizers==0.21.1
+opendelta==0.3.2
+pycparser==2.22
+narwhals==1.41.0
+scikit-learn==1.6.1
+dill==0.3.8
+oss2==2.15.0
+yacs==0.1.8
+more-itertools==10.7.0
+pip==25.1.1
+threadpoolctl==3.6.0
+flash-attn==2.7.1.post1
+bigmodelvis==0.0.1
+pathlib==1.0.1
+delta-center-client==0.0.4
+xxhash==3.5.0
+wandb==0.19.11
+setproctitle==1.3.6
+aliyun-python-sdk-core==2.16.0
+transformers==4.52.3
+aliyun-python-sdk-kms==2.16.5
+datasets==3.6.0
+typer==0.16.0
+docker-pycreds==0.4.0
+click==8.2.1
+huggingface-hub==0.32.1
+web.py==0.62
+cffi==1.17.1
+opencv-python==4.11.0.86
+jsonschema==4.24.0
+typing_extensions==4.12.2
+jaraco.functools==4.0.1
+jaraco.text==3.12.1
+jaraco.collections==5.1.0
+inflect==7.3.1
+more-itertools==10.3.0
+packaging==24.2
+importlib_metadata==8.0.0
+backports.tarfile==1.2.0
+typeguard==4.3.0
+zipp==3.19.2
+platformdirs==4.2.2
+autocommand==2.2.2
+jaraco.context==5.3.0
+tomli==2.0.1
+wheel==0.45.1

ProtT3/all_checkpoints/stage1_06290009_deepspeed/wandb/run-20250629_001200-vgvxxzqc/files/wandb-metadata.json

@@ -0,0 +1,97 @@
+{
+  "os": "Linux-5.10.134-008.16.kangaroo.al8.x86_64-x86_64-with-glibc2.35",
+  "python": "CPython 3.10.0",
+  "startedAt": "2025-06-28T16:12:00.926076Z",
+  "args": [
+    "--devices",
+    "0,1,2,3,4,5,6,7",
+    "--mode",
+    "train",
+    "--filename",
+    "stage1_06290009_deepspeed",
+    "--num_query_token",
+    "8",
+    "--plm_name",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m",
+    "--bert_name",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft",
+    "--save_every_n_epochs",
+    "5",
+    "--max_epochs",
+    "20",
+    "--batch_size",
+    "168",
+    "--precision",
+    "bf16-mixed",
+    "--mix_dataset",
+    "--num_workers",
+    "8",
+    "--use_wandb_logger"
+  ],
+  "program": "/nas/shared/kilab/wangyujia/ProtT3/stage1.py",
+  "codePath": "stage1.py",
+  "root": "./all_checkpoints/stage1_06290009_deepspeed/",
+  "host": "dsw-251511-c65bb988c-9g24f",
+  "executable": "/root/miniconda3/envs/protT3/bin/python",
+  "codePathLocal": "stage1.py",
+  "cpu_count": 64,
+  "cpu_count_logical": 64,
+  "gpu": "NVIDIA A800-SXM4-80GB",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1623302262784",
+      "used": "987680768"
+    }
+  },
+  "memory": {
+    "total": "549755813888"
+  },
+  "cpu": {
+    "count": 64,
+    "countLogical": 64
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    }
+  ],
+  "cudaVersion": "12.1"
+}

ProtT3/all_checkpoints/stage1_06290009_deepspeed/wandb/run-20250629_001200-vgvxxzqc/files/wandb-summary.json

@@ -0,0 +1 @@
+{"loader1/val_loss_lm/dataloader_idx_1":5.460818767547607,"lr":1.0554024811426643e-05,"_timestamp":1.7511476307831557e+09,"loader1/val_loss/dataloader_idx_1":14.773687362670898,"train_loss":2.630859375,"loader2/val_loss_ptm/dataloader_idx_2":2.988206148147583,"loader1/val_loss_ptm/dataloader_idx_1":3.491912603378296,"loader2/val_loss_lm/dataloader_idx_2":6.094737529754639,"_runtime":20509.857471191,"loader2/val_loss_ptc/dataloader_idx_2":5.59119987487793,"_wandb":{"runtime":20517},"train_loss_ptc":0.64306640625,"train_loss_ptm":0.291015625,"loader0/val_loss_ptc/dataloader_idx_0":1.0085703134536743,"loader1/val_loss_ptc/dataloader_idx_1":5.825062274932861,"loader0/val_loss_lm/dataloader_idx_0":2.1440062522888184,"loader2/val_loss/dataloader_idx_2":14.674175262451172,"epoch":19,"train_loss_lm":1.697265625,"_step":147,"loader0/val_loss/dataloader_idx_0":3.6699562072753906,"loader0/val_loss_ptm/dataloader_idx_0":0.5172882676124573,"trainer/global_step":6399}

ProtT3/all_checkpoints/stage1_06290009_deepspeed/wandb/run-20250629_001200-vgvxxzqc/logs/debug-internal.log

@@ -0,0 +1,42 @@
+{"time":"2025-06-29T00:12:00.930911849+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.11","symlink path":"all_checkpoints/stage1_06290009_deepspeed/wandb/run-20250629_001200-vgvxxzqc/logs/debug-core.log"}
+{"time":"2025-06-29T00:12:31.03707486+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T00:12:36.876219526+08:00","level":"INFO","msg":"created new stream","id":"vgvxxzqc"}
+{"time":"2025-06-29T00:12:36.876272436+08:00","level":"INFO","msg":"stream: started","id":"vgvxxzqc"}
+{"time":"2025-06-29T00:12:36.876317878+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"vgvxxzqc"}
+{"time":"2025-06-29T00:12:36.876360145+08:00","level":"INFO","msg":"handler: started","stream_id":"vgvxxzqc"}
+{"time":"2025-06-29T00:12:36.876401621+08:00","level":"INFO","msg":"sender: started","stream_id":"vgvxxzqc"}
+{"time":"2025-06-29T00:12:39.839397838+08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2025-06-29T00:13:20.87652364+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:58014->104.21.20.172:443: read: connection reset by peer"}
+{"time":"2025-06-29T00:16:17.426969211+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": unexpected EOF"}
+{"time":"2025-06-29T00:17:58.51721346+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": unexpected EOF"}
+{"time":"2025-06-29T00:26:10.222503445+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:51478->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-06-29T00:28:34.807681677+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:46066->104.21.20.172:443: read: connection reset by peer"}
+{"time":"2025-06-29T00:33:01.870533298+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:55734->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-29T00:33:23.754842161+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:46042->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-06-29T00:36:47.149592254+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:57176->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-06-29T00:41:05.645712211+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": unexpected EOF"}
+{"time":"2025-06-29T00:47:20.493525378+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:58300->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-29T00:47:45.629981285+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": unexpected EOF"}
+{"time":"2025-06-29T00:51:19.597480154+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:60802->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-29T00:54:09.777365701+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:51532->104.21.20.172:443: read: connection reset by peer"}
+{"time":"2025-06-29T01:00:20.78154967+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:34934->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-06-29T01:04:21.421531776+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:55938->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-29T01:05:41.05509194+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:45678->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-06-29T01:08:44.130266043+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": unexpected EOF"}
+{"time":"2025-06-29T01:10:59.724692621+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": unexpected EOF"}
+{"time":"2025-06-29T01:14:35.821570745+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:35232->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-29T01:17:35.533530754+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:44258->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-29T01:23:12.630381225+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:48108->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-06-29T01:24:55.067569821+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-29T01:25:27.188065501+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-06-29T01:25:56.782489942+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:45024->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-29T01:26:49.538892815+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": unexpected EOF"}
+{"time":"2025-06-29T01:29:46.157546022+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06290009_deepspeed/vgvxxzqc/file_stream\": read tcp 10.1.7.100:42508->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-06-29T05:53:57.9412544+08:00","level":"INFO","msg":"stream: closing","id":"vgvxxzqc"}
+{"time":"2025-06-29T05:53:57.941286983+08:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2025-06-29T05:53:57.942366437+08:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2025-06-29T05:54:00.869660002+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-06-29T05:54:03.731237694+08:00","level":"INFO","msg":"handler: closed","stream_id":"vgvxxzqc"}
+{"time":"2025-06-29T05:54:03.731282348+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"vgvxxzqc"}
+{"time":"2025-06-29T05:54:03.731313818+08:00","level":"INFO","msg":"sender: closed","stream_id":"vgvxxzqc"}
+{"time":"2025-06-29T05:54:03.735031072+08:00","level":"INFO","msg":"stream: closed","id":"vgvxxzqc"}

ProtT3/all_checkpoints/stage1_06290009_deepspeed/wandb/run-20250629_001200-vgvxxzqc/logs/debug.log

@@ -0,0 +1,24 @@
+2025-06-29 00:12:00,920 INFO    MainThread:2351 [wandb_setup.py:_flush():70] Current SDK version is 0.19.11
+2025-06-29 00:12:00,920 INFO    MainThread:2351 [wandb_setup.py:_flush():70] Configure stats pid to 2351
+2025-06-29 00:12:00,920 INFO    MainThread:2351 [wandb_setup.py:_flush():70] Loading settings from /root/.config/wandb/settings
+2025-06-29 00:12:00,920 INFO    MainThread:2351 [wandb_setup.py:_flush():70] Loading settings from /nas/shared/kilab/wangyujia/ProtT3/wandb/settings
+2025-06-29 00:12:00,920 INFO    MainThread:2351 [wandb_setup.py:_flush():70] Loading settings from environment variables
+2025-06-29 00:12:00,920 INFO    MainThread:2351 [wandb_init.py:setup_run_log_directory():724] Logging user logs to ./all_checkpoints/stage1_06290009_deepspeed/wandb/run-20250629_001200-vgvxxzqc/logs/debug.log
+2025-06-29 00:12:00,920 INFO    MainThread:2351 [wandb_init.py:setup_run_log_directory():725] Logging internal logs to ./all_checkpoints/stage1_06290009_deepspeed/wandb/run-20250629_001200-vgvxxzqc/logs/debug-internal.log
+2025-06-29 00:12:00,920 INFO    MainThread:2351 [wandb_init.py:init():852] calling init triggers
+2025-06-29 00:12:00,921 INFO    MainThread:2351 [wandb_init.py:init():857] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-06-29 00:12:00,921 INFO    MainThread:2351 [wandb_init.py:init():893] starting backend
+2025-06-29 00:12:00,921 INFO    MainThread:2351 [wandb_init.py:init():897] sending inform_init request
+2025-06-29 00:12:00,923 INFO    MainThread:2351 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-06-29 00:12:00,924 INFO    MainThread:2351 [wandb_init.py:init():907] backend started and connected
+2025-06-29 00:12:00,926 INFO    MainThread:2351 [wandb_init.py:init():1005] updated telemetry
+2025-06-29 00:12:00,929 INFO    MainThread:2351 [wandb_init.py:init():1029] communicating run to backend with 90.0 second timeout
+2025-06-29 00:12:39,788 INFO    MainThread:2351 [wandb_init.py:init():1104] starting run threads in backend
+2025-06-29 00:12:40,030 INFO    MainThread:2351 [wandb_run.py:_console_start():2573] atexit reg
+2025-06-29 00:12:40,030 INFO    MainThread:2351 [wandb_run.py:_redirect():2421] redirect: wrap_raw
+2025-06-29 00:12:40,034 INFO    MainThread:2351 [wandb_run.py:_redirect():2490] Wrapping output streams.
+2025-06-29 00:12:40,034 INFO    MainThread:2351 [wandb_run.py:_redirect():2513] Redirects installed.
+2025-06-29 00:12:40,036 INFO    MainThread:2351 [wandb_init.py:init():1150] run started, returning control to user process
+2025-06-29 00:12:46,669 INFO    MainThread:2351 [wandb_run.py:_config_callback():1436] config_cb None None {'filename': 'stage1_06290009_deepspeed', 'seed': 42, 'mode': 'train', 'strategy': 'deepspeed', 'accelerator': 'gpu', 'devices': '0,1,2,3,4,5,6,7', 'precision': 'bf16-mixed', 'max_epochs': 20, 'check_val_every_n_epoch': 1, 'use_wandb_logger': True, 'mix_dataset': True, 'temperature': 0.1, 'save_every_n_epochs': 5, 'ptm': True, 'lm': True, 'rerank_cand_num': 128, 'plm_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m', 'plm_tune': 'freeze', 'load_4bit': False, 'pool_size': 0, 'bert_hidden_dim': 768, 'bert_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft', 'projection_dim': 256, 'cross_attention_freq': 2, 'num_query_token': 8, 'weight_decay': 0.05, 'init_lr': 0.0001, 'min_lr': 1e-05, 'warmup_lr': 1e-06, 'warmup_steps': 1000, 'lr_decay_rate': 0.9, 'scheduler': 'linear_warmup_cosine_lr', 'init_checkpoint': '', 'retrieval_eval_epoch': 10, 'num_workers': 8, 'batch_size': 168, 'match_batch_size': 64, 'root': 'data', 'text_max_len': 128, 'prot_max_len': 1024, 'prot_aug': 'None'}
+2025-06-29 05:53:57,929 INFO    MsgRouterThr:2351 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 2 handles.

ProtT3/all_checkpoints/stage1_07041727_2dataset/wandb/debug-internal.log

@@ -0,0 +1,87 @@
+{"time":"2025-07-04T17:28:55.160594539+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.11","symlink path":"all_checkpoints/stage1_07041727_2dataset/wandb/run-20250704_172854-6bkqzmou/logs/debug-core.log"}
+{"time":"2025-07-04T17:28:56.249328974+08:00","level":"INFO","msg":"created new stream","id":"6bkqzmou"}
+{"time":"2025-07-04T17:28:56.249372351+08:00","level":"INFO","msg":"stream: started","id":"6bkqzmou"}
+{"time":"2025-07-04T17:28:56.249400451+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"6bkqzmou"}
+{"time":"2025-07-04T17:28:56.249431272+08:00","level":"INFO","msg":"sender: started","stream_id":"6bkqzmou"}
+{"time":"2025-07-04T17:28:56.249469216+08:00","level":"INFO","msg":"handler: started","stream_id":"6bkqzmou"}
+{"time":"2025-07-04T17:28:57.491653525+08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2025-07-04T22:06:52.200518707+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": unexpected EOF"}
+{"time":"2025-07-04T22:09:45.338273816+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": unexpected EOF"}
+{"time":"2025-07-04T22:19:20.574743081+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:59078->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-04T22:25:54.288016702+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:42132->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-04T22:29:40.591991523+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:37544->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-04T22:36:54.256091094+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:40968->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-04T22:37:22.364944108+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": unexpected EOF"}
+{"time":"2025-07-04T22:40:24.499117928+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-07-04T22:40:51.249223858+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": unexpected EOF"}
+{"time":"2025-07-04T22:44:05.872015851+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:59778->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-04T22:49:18.192032141+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:47358->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-07-04T22:52:13.295997002+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:46182->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-04T22:53:26.345699486+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": unexpected EOF"}
+{"time":"2025-07-04T22:55:37.691524069+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": unexpected EOF"}
+{"time":"2025-07-04T22:59:01.477384402+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": unexpected EOF"}
+{"time":"2025-07-04T23:01:22.224282887+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:45222->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-07-04T23:06:44.720013857+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:46280->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-07-04T23:08:42.894770628+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-07-04T23:10:13.616061547+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:57034->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-04T23:11:27.896127402+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-07-04T23:15:47.19805854+08:00","level":"ERROR","msg":"filestream: json decode error: net/http: request canceled (Client.Timeout or context cancellation while reading body)"}
+{"time":"2025-07-04T23:15:47.222866077+08:00","level":"ERROR","msg":"filestream: error closing response body: net/http: request canceled"}
+{"time":"2025-07-04T23:19:26.063989295+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:35504->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-04T23:21:57.905369451+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-07-04T23:23:00.080992848+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:38378->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-04T23:26:54.577250259+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:41928->104.21.20.172:443: read: connection reset by peer"}
+{"time":"2025-07-04T23:28:47.703904029+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:38040->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-07-04T23:30:12.910139882+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-07-04T23:30:45.313312591+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-07-04T23:35:25.039973358+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:38402->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-07-04T23:39:27.49206097+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:39926->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-07-04T23:43:09.424012888+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:59448->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-04T23:46:07.600020006+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:33610->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-04T23:46:28.951111977+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": unexpected EOF"}
+{"time":"2025-07-04T23:48:12.919414088+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-07-04T23:48:45.403207458+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-07-04T23:49:16.527984782+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:49472->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-04T23:50:27.921623046+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-07-04T23:52:57.899934024+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:40636->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-07-04T23:56:18.928962652+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:53056->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-04T23:56:57.924908638+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-07-04T23:57:30.375318804+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-07-04T23:58:04.586629939+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-07-05T00:03:09.552010125+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:39630->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-05T00:03:42.930344983+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-07-05T00:04:15.375941679+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-07-05T00:04:21.655905995+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:48150->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-07-05T00:04:50.22664016+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-07-05T00:08:01.391966638+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:53260->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-07-05T00:12:23.023992865+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:44698->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-05T00:15:42.93731147+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-07-05T00:16:15.214992648+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-07-05T00:16:49.667525584+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-07-05T00:17:01.040050871+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:42760->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-05T00:20:02.288062562+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:37934->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-07-05T00:23:13.264033499+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:51502->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-05T00:26:32.944001316+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:50460->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-05T00:30:17.199971228+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:43756->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-07-05T00:35:42.94748626+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-07-05T00:35:44.881161178+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:59610->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-05T00:39:14.287974585+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:46688->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-07-05T00:39:33.427697791+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:40430->104.21.20.172:443: read: connection reset by peer"}
+{"time":"2025-07-05T00:40:47.648388331+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:60164->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-07-05T00:42:34.088456552+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:52978->104.21.20.172:443: read: connection reset by peer"}
+{"time":"2025-07-05T00:47:13.006425282+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:33354->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-07-05T00:49:55.823998082+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:33330->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-05T00:52:51.439993456+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:52416->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-05T00:57:07.440983899+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:39188->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-05T01:04:51.312039238+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:59436->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-05T01:09:40.080000713+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:34348->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-05T01:18:31.535996696+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:57184->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-07-05T01:25:12.431983593+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:59852->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-05T11:36:49.210079644+08:00","level":"INFO","msg":"stream: closing","id":"6bkqzmou"}
+{"time":"2025-07-05T11:36:49.210163239+08:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2025-07-05T11:36:49.211103046+08:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2025-07-05T11:36:51.804545543+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-07-05T11:36:53.755788884+08:00","level":"INFO","msg":"handler: closed","stream_id":"6bkqzmou"}
+{"time":"2025-07-05T11:36:53.755828602+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"6bkqzmou"}
+{"time":"2025-07-05T11:36:53.75584333+08:00","level":"INFO","msg":"sender: closed","stream_id":"6bkqzmou"}
+{"time":"2025-07-05T11:36:53.759902053+08:00","level":"INFO","msg":"stream: closed","id":"6bkqzmou"}

ProtT3/all_checkpoints/stage1_07041727_2dataset/wandb/debug.log

@@ -0,0 +1,24 @@
+2025-07-04 17:28:55,100 INFO    MainThread:29356 [wandb_setup.py:_flush():70] Current SDK version is 0.19.11
+2025-07-04 17:28:55,100 INFO    MainThread:29356 [wandb_setup.py:_flush():70] Configure stats pid to 29356
+2025-07-04 17:28:55,100 INFO    MainThread:29356 [wandb_setup.py:_flush():70] Loading settings from /root/.config/wandb/settings
+2025-07-04 17:28:55,100 INFO    MainThread:29356 [wandb_setup.py:_flush():70] Loading settings from /nas/shared/kilab/wangyujia/ProtT3/wandb/settings
+2025-07-04 17:28:55,100 INFO    MainThread:29356 [wandb_setup.py:_flush():70] Loading settings from environment variables
+2025-07-04 17:28:55,100 INFO    MainThread:29356 [wandb_init.py:setup_run_log_directory():724] Logging user logs to ./all_checkpoints/stage1_07041727_2dataset/wandb/run-20250704_172854-6bkqzmou/logs/debug.log
+2025-07-04 17:28:55,100 INFO    MainThread:29356 [wandb_init.py:setup_run_log_directory():725] Logging internal logs to ./all_checkpoints/stage1_07041727_2dataset/wandb/run-20250704_172854-6bkqzmou/logs/debug-internal.log
+2025-07-04 17:28:55,100 INFO    MainThread:29356 [wandb_init.py:init():852] calling init triggers
+2025-07-04 17:28:55,100 INFO    MainThread:29356 [wandb_init.py:init():857] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-07-04 17:28:55,100 INFO    MainThread:29356 [wandb_init.py:init():893] starting backend
+2025-07-04 17:28:55,100 INFO    MainThread:29356 [wandb_init.py:init():897] sending inform_init request
+2025-07-04 17:28:55,102 INFO    MainThread:29356 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-07-04 17:28:55,102 INFO    MainThread:29356 [wandb_init.py:init():907] backend started and connected
+2025-07-04 17:28:55,103 INFO    MainThread:29356 [wandb_init.py:init():1005] updated telemetry
+2025-07-04 17:28:55,103 INFO    MainThread:29356 [wandb_init.py:init():1029] communicating run to backend with 90.0 second timeout
+2025-07-04 17:28:57,453 INFO    MainThread:29356 [wandb_init.py:init():1104] starting run threads in backend
+2025-07-04 17:28:57,668 INFO    MainThread:29356 [wandb_run.py:_console_start():2573] atexit reg
+2025-07-04 17:28:57,669 INFO    MainThread:29356 [wandb_run.py:_redirect():2421] redirect: wrap_raw
+2025-07-04 17:28:57,678 INFO    MainThread:29356 [wandb_run.py:_redirect():2490] Wrapping output streams.
+2025-07-04 17:28:57,684 INFO    MainThread:29356 [wandb_run.py:_redirect():2513] Redirects installed.
+2025-07-04 17:28:57,686 INFO    MainThread:29356 [wandb_init.py:init():1150] run started, returning control to user process
+2025-07-04 17:29:03,015 INFO    MainThread:29356 [wandb_run.py:_config_callback():1436] config_cb None None {'filename': 'stage1_07041727_2dataset', 'seed': 42, 'mode': 'train', 'strategy': 'deepspeed', 'accelerator': 'gpu', 'devices': '0,1,2,3,4,5,6,7', 'precision': 'bf16-mixed', 'max_epochs': 30, 'check_val_every_n_epoch': 1, 'use_wandb_logger': True, 'mix_dataset': True, 'temperature': 0.1, 'save_every_n_epochs': 5, 'ptm': True, 'lm': True, 'rerank_cand_num': 128, 'plm_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m', 'plm_tune': 'freeze', 'load_4bit': False, 'pool_size': 0, 'bert_hidden_dim': 768, 'bert_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft', 'projection_dim': 256, 'cross_attention_freq': 2, 'num_query_token': 8, 'weight_decay': 0.05, 'init_lr': 0.0001, 'min_lr': 1e-05, 'warmup_lr': 1e-06, 'warmup_steps': 1000, 'lr_decay_rate': 0.9, 'scheduler': 'linear_warmup_cosine_lr', 'init_checkpoint': '', 'retrieval_eval_epoch': 10, 'num_workers': 8, 'batch_size': 32, 'match_batch_size': 64, 'root': 'data', 'text_max_len': 128, 'prot_max_len': 1024, 'prot_aug': 'None'}
+2025-07-05 11:36:49,208 INFO    MsgRouterThr:29356 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles.

ProtT3/all_checkpoints/stage1_07041727_2dataset/wandb/run-20250704_172854-6bkqzmou/files/config.yaml

@@ -0,0 +1,236 @@
+_wandb:
+    value:
+        cli_version: 0.19.11
+        m:
+            - "1": loader1/val_loss_lm/dataloader_idx_1
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": trainer/global_step
+              "6":
+                - 3
+              "7": []
+            - "1": loader2/val_loss/dataloader_idx_2
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": lr
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loader1/val_loss/dataloader_idx_1
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loader2/val_loss_ptc/dataloader_idx_2
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loader1/val_loss_ptm/dataloader_idx_1
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loader2/val_loss_ptm/dataloader_idx_2
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": train_loss
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loader0/val_loss_ptc/dataloader_idx_0
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loader0/val_loss_ptm/dataloader_idx_0
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loader1/val_loss_ptc/dataloader_idx_1
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loader0/val_loss_lm/dataloader_idx_0
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": train_loss_ptc
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": train_loss_ptm
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": train_loss_lm
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": epoch
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loader0/val_loss/dataloader_idx_0
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loader2/val_loss_lm/dataloader_idx_2
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+        python_version: 3.10.0
+        t:
+            "1":
+                - 1
+                - 5
+                - 9
+                - 11
+                - 33
+                - 41
+                - 49
+                - 53
+                - 55
+                - 63
+                - 103
+            "2":
+                - 1
+                - 5
+                - 9
+                - 11
+                - 33
+                - 41
+                - 49
+                - 53
+                - 55
+                - 63
+                - 103
+            "3":
+                - 7
+                - 23
+                - 55
+                - 66
+            "4": 3.10.0
+            "5": 0.19.11
+            "6": 4.52.3
+            "8":
+                - 5
+            "12": 0.19.11
+            "13": linux-x86_64
+accelerator:
+    value: gpu
+batch_size:
+    value: 32
+bert_hidden_dim:
+    value: 768
+bert_name:
+    value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft
+check_val_every_n_epoch:
+    value: 1
+cross_attention_freq:
+    value: 2
+devices:
+    value: 0,1,2,3,4,5,6,7
+filename:
+    value: stage1_07041727_2dataset
+init_checkpoint:
+    value: ""
+init_lr:
+    value: 0.0001
+lm:
+    value: true
+load_4bit:
+    value: false
+lr_decay_rate:
+    value: 0.9
+match_batch_size:
+    value: 64
+max_epochs:
+    value: 30
+min_lr:
+    value: 1e-05
+mix_dataset:
+    value: true
+mode:
+    value: train
+num_query_token:
+    value: 8
+num_workers:
+    value: 8
+plm_name:
+    value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m
+plm_tune:
+    value: freeze
+pool_size:
+    value: 0
+precision:
+    value: bf16-mixed
+projection_dim:
+    value: 256
+prot_aug:
+    value: None
+prot_max_len:
+    value: 1024
+ptm:
+    value: true
+rerank_cand_num:
+    value: 128
+retrieval_eval_epoch:
+    value: 10
+root:
+    value: data
+save_every_n_epochs:
+    value: 5
+scheduler:
+    value: linear_warmup_cosine_lr
+seed:
+    value: 42
+strategy:
+    value: deepspeed
+temperature:
+    value: 0.1
+text_max_len:
+    value: 128
+use_wandb_logger:
+    value: true
+warmup_lr:
+    value: 1e-06
+warmup_steps:
+    value: 1000
+weight_decay:
+    value: 0.05

ProtT3/all_checkpoints/stage1_07041727_2dataset/wandb/run-20250704_172854-6bkqzmou/files/output.log

@@ -0,0 +1,21 @@
+/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:654: Checkpoint directory /nas/shared/kilab/wangyujia/ProtT3/all_checkpoints/stage1_07041727_2dataset exists and is not empty.
+Enabling DeepSpeed BF16. Model parameters and inputs will be cast to `bfloat16`.
+LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]
+
+  | Name         | Type         | Params | Mode
+------------------------------------------------------
+0 | blip2qformer | Blip2Qformer | 327 M  | train
+------------------------------------------------------
+179 M     Trainable params
+147 M     Non-trainable params
+327 M     Total params
+1,309.467 Total estimated model params size (MB)
+5         Modules in train mode
+926       Modules in eval mode
+Epoch 29: 100%|█████████████████████████████████████████| 3331/3331 [36:16<00:00,  1.53it/s, v_num=zmou]
+/nas/shared/kilab/wangyujia/ProtT3/model/blip2qformer.py:220: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(enable_autocast, dtype=torch.float32):
+                                                                                                        
+/nas/shared/kilab/wangyujia/ProtT3/model/dist_funs.py:18: FutureWarning: Positional args are being deprecated, use kwargs instead. Refer to https://pytorch.org/docs/main/generated/torch.nn.Module.html#torch.nn.Module.state_dict for details.
+  sd = self.module.state_dict(destination, prefix, keep_vars)
+`Trainer.fit` stopped: `max_epochs=30` reached.

ProtT3/all_checkpoints/stage1_07041727_2dataset/wandb/run-20250704_172854-6bkqzmou/files/requirements.txt

@@ -0,0 +1,225 @@
+attrs==25.3.0
+tqdm==4.67.1
+langcodes==3.5.0
+nvidia-cublas-cu12==12.4.5.8
+tifffile==2025.5.10
+nvidia-cufile-cu12==1.11.1.6
+nltk==3.9.1
+salesforce-lavis==1.0.2
+tzdata==2025.2
+pyparsing==3.2.3
+six==1.17.0
+python-dateutil==2.9.0.post0
+pandas==2.2.3
+pytorch-lightning==2.5.1.post0
+blinker==1.9.0
+opencv-python-headless==4.5.5.64
+nvidia-cusparse-cu12==12.3.1.170
+pytz==2025.2
+async-timeout==5.0.1
+pillow==11.2.1
+parso==0.8.4
+joblib==1.5.1
+contourpy==1.3.2
+triton==3.2.0
+marisa-trie==1.2.1
+PyYAML==6.0.2
+regex==2024.11.6
+idna==3.10
+nvidia-curand-cu12==10.3.5.147
+rpds-py==0.25.1
+aiosignal==1.3.2
+srsly==2.5.1
+confection==0.1.5
+typing-inspection==0.4.1
+packaging==24.2
+distlib==0.3.9
+networkx==3.4.2
+absl-py==2.2.2
+yarl==1.20.0
+lightning-utilities==0.14.3
+executing==2.2.0
+pycocoevalcap==1.2
+wheel==0.45.1
+nvidia-ml-py==12.575.51
+cycler==0.12.1
+wrapt==1.17.2
+jsonschema-specifications==2025.4.1
+protobuf==6.31.0
+mpmath==1.3.0
+certifi==2025.4.26
+py-cpuinfo==9.0.0
+contexttimer==0.3.3
+watchdog==6.0.0
+pexpect==4.9.0
+webencodings==0.5.1
+hf-xet==1.1.2
+cymem==2.0.11
+requests==2.32.3
+timm==0.4.12
+omegaconf==2.3.0
+nvidia-nvjitlink-cu12==12.4.127
+webdataset==0.2.111
+nodeenv==1.9.1
+frozenlist==1.6.0
+annotated-types==0.7.0
+matplotlib-inline==0.1.7
+urllib3==2.4.0
+rich==14.0.0
+GitPython==3.1.44
+lazy_loader==0.4
+msgpack==1.1.0
+prompt_toolkit==3.0.51
+fonttools==4.58.0
+multidict==6.4.4
+blis==1.3.0
+thinc==8.3.6
+nvidia-nvtx-cu12==12.4.127
+torchmetrics==1.7.1
+weasel==0.4.1
+numpy==2.2.6
+cachetools==5.5.2
+Jinja2==3.1.6
+matplotlib==3.10.3
+nvidia-cudnn-cu12==9.1.0.70
+Pygments==2.19.1
+tornado==6.5.1
+scipy==1.15.3
+rouge_score==0.1.2
+cloudpathlib==0.21.1
+jedi==0.19.2
+referencing==0.36.2
+decord==0.6.0
+setuptools==78.1.1
+mdurl==0.1.2
+identify==2.6.12
+python-slugify==8.0.4
+portalocker==3.1.1
+catalogue==2.0.10
+platformdirs==4.3.8
+antlr4-python3-runtime==4.9.3
+nvidia-cusolver-cu12==11.6.1.9
+kaggle==1.7.4.5
+pydeck==0.9.1
+pydantic==2.11.5
+nvidia-cufft-cu12==11.2.1.3
+pyarrow==20.0.0
+nvidia-nccl-cu12==2.21.5
+markdown-it-py==3.0.0
+gitdb==4.0.12
+altair==5.5.0
+torchvision==0.21.0
+python-magic==0.4.27
+iopath==0.1.10
+smart-open==7.1.0
+torch==2.6.0
+pycocotools==2.0.8
+fairscale==0.4.4
+traitlets==5.14.3
+pure_eval==0.2.3
+sympy==1.13.1
+nvidia-cusparselt-cu12==0.6.2
+imageio==2.37.0
+stack-data==0.6.3
+shellingham==1.5.4
+nvidia-cuda-runtime-cu12==12.4.127
+einops==0.8.1
+tenacity==9.1.2
+virtualenv==20.31.2
+ptyprocess==0.7.0
+cfgv==3.4.0
+pre_commit==4.2.0
+language_data==1.3.0
+typing_extensions==4.13.2
+propcache==0.3.1
+nvidia-cuda-cupti-cu12==12.4.127
+safetensors==0.5.3
+text-unidecode==1.3
+wcwidth==0.2.13
+charset-normalizer==3.4.2
+aiohappyeyeballs==2.6.1
+ipython==8.36.0
+streamlit==1.45.1
+asttokens==3.0.0
+psutil==7.0.0
+smmap==5.0.2
+exceptiongroup==1.3.0
+murmurhash==1.0.13
+filelock==3.18.0
+plotly==6.1.1
+hjson==3.1.0
+pydantic_core==2.33.2
+ninja==1.11.1.4
+kiwisolver==1.4.8
+spacy-legacy==3.0.12
+opendatasets==0.1.22
+decorator==5.2.1
+spacy==3.8.7
+wasabi==1.1.3
+sentencepiece==0.2.0
+toml==0.10.2
+scikit-image==0.25.2
+deepspeed==0.16.10+b666844f
+ftfy==6.3.1
+bleach==6.2.0
+nvidia-cuda-nvrtc-cu12==12.4.127
+spacy-loggers==1.0.5
+MarkupSafe==3.0.2
+braceexpand==0.1.7
+oss2==2.15.0
+preshed==3.0.10
+transformers==4.52.3
+aiohttp==3.12.2
+web.py==0.62
+threadpoolctl==3.6.0
+jaraco.functools==4.1.0
+wandb==0.19.11
+sentry-sdk==2.29.1
+tokenizers==0.21.1
+fsspec==2025.3.0
+flash-attn==2.7.1.post1
+opendelta==0.3.2
+opencv-python==4.11.0.86
+click==8.2.1
+docker-pycreds==0.4.0
+typer==0.16.0
+xxhash==3.5.0
+pathlib==1.0.1
+dill==0.3.8
+crcmod==1.7
+bigmodelvis==0.0.1
+datasets==3.6.0
+pycryptodome==3.23.0
+jsonschema==4.24.0
+aliyun-python-sdk-core==2.16.0
+jmespath==0.10.0
+more-itertools==10.7.0
+scikit-learn==1.6.1
+huggingface-hub==0.32.1
+cryptography==45.0.3
+pycparser==2.22
+yacs==0.1.8
+aliyun-python-sdk-kms==2.16.5
+cffi==1.17.1
+delta-center-client==0.0.4
+multiprocess==0.70.16
+setproctitle==1.3.6
+narwhals==1.41.0
+pip==25.1.1
+cheroot==10.0.1
+jaraco.context==5.3.0
+more-itertools==10.3.0
+jaraco.functools==4.0.1
+jaraco.text==3.12.1
+platformdirs==4.2.2
+packaging==24.2
+wheel==0.45.1
+zipp==3.19.2
+inflect==7.3.1
+autocommand==2.2.2
+typeguard==4.3.0
+jaraco.collections==5.1.0
+backports.tarfile==1.2.0
+tomli==2.0.1
+importlib_metadata==8.0.0
+typing_extensions==4.12.2

ProtT3/all_checkpoints/stage1_07041727_2dataset/wandb/run-20250704_172854-6bkqzmou/files/wandb-metadata.json

@@ -0,0 +1,98 @@
+{
+  "os": "Linux-5.10.134-008.16.kangaroo.al8.x86_64-x86_64-with-glibc2.35",
+  "python": "CPython 3.10.0",
+  "startedAt": "2025-07-04T09:28:55.102499Z",
+  "args": [
+    "--devices",
+    "0,1,2,3,4,5,6,7",
+    "--mode",
+    "train",
+    "--filename",
+    "stage1_07041727_2dataset",
+    "--num_query_token",
+    "8",
+    "--plm_name",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m",
+    "--bert_name",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft",
+    "--save_every_n_epochs",
+    "5",
+    "--max_epochs",
+    "30",
+    "--batch_size",
+    "32",
+    "--precision",
+    "bf16-mixed",
+    "--mix_dataset",
+    "--num_workers",
+    "8",
+    "--use_wandb_logger"
+  ],
+  "program": "/nas/shared/kilab/wangyujia/ProtT3/stage1.py",
+  "codePath": "stage1.py",
+  "email": "gia0603yucca@gmail.com",
+  "root": "./all_checkpoints/stage1_07041727_2dataset/",
+  "host": "dsw-266702-5bd8569444-hrqd7",
+  "executable": "/root/miniconda3/envs/protT3/bin/python",
+  "codePathLocal": "stage1.py",
+  "cpu_count": 64,
+  "cpu_count_logical": 64,
+  "gpu": "NVIDIA A800-SXM4-80GB",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1623302262784",
+      "used": "1260302336"
+    }
+  },
+  "memory": {
+    "total": "549755813888"
+  },
+  "cpu": {
+    "count": 64,
+    "countLogical": 64
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    }
+  ],
+  "cudaVersion": "12.1"
+}

ProtT3/all_checkpoints/stage1_07041727_2dataset/wandb/run-20250704_172854-6bkqzmou/files/wandb-summary.json

@@ -0,0 +1 @@
+{"loader2/val_loss/dataloader_idx_2":13.163299560546875,"loader2/val_loss_ptm/dataloader_idx_2":2.123929738998413,"_step":2027,"loader1/val_loss_ptc/dataloader_idx_1":0.5523239970207214,"lr":1.0246513738820795e-05,"epoch":29,"_runtime":65267.979623679,"loader0/val_loss_ptm/dataloader_idx_0":0.3546168804168701,"train_loss_lm":1.42578125,"train_loss_ptc":0.218505859375,"loader0/val_loss/dataloader_idx_0":2.6084561347961426,"_timestamp":1.7516866030817497e+09,"loader2/val_loss_lm/dataloader_idx_2":7.963825225830078,"_wandb":{"runtime":65274},"loader1/val_loss/dataloader_idx_1":2.692500114440918,"loader1/val_loss_lm/dataloader_idx_1":1.6366312503814697,"loader2/val_loss_ptc/dataloader_idx_2":3.075716018676758,"train_loss_ptm":0.2022705078125,"loader0/val_loss_lm/dataloader_idx_0":1.9205952882766724,"loader1/val_loss_ptm/dataloader_idx_1":0.50240159034729,"loader0/val_loss_ptc/dataloader_idx_0":0.3330143988132477,"trainer/global_step":99929,"train_loss":1.845703125}

ProtT3/all_checkpoints/stage1_07041727_2dataset/wandb/run-20250704_172854-6bkqzmou/logs/debug-internal.log

@@ -0,0 +1,87 @@
+{"time":"2025-07-04T17:28:55.160594539+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.11","symlink path":"all_checkpoints/stage1_07041727_2dataset/wandb/run-20250704_172854-6bkqzmou/logs/debug-core.log"}
+{"time":"2025-07-04T17:28:56.249328974+08:00","level":"INFO","msg":"created new stream","id":"6bkqzmou"}
+{"time":"2025-07-04T17:28:56.249372351+08:00","level":"INFO","msg":"stream: started","id":"6bkqzmou"}
+{"time":"2025-07-04T17:28:56.249400451+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"6bkqzmou"}
+{"time":"2025-07-04T17:28:56.249431272+08:00","level":"INFO","msg":"sender: started","stream_id":"6bkqzmou"}
+{"time":"2025-07-04T17:28:56.249469216+08:00","level":"INFO","msg":"handler: started","stream_id":"6bkqzmou"}
+{"time":"2025-07-04T17:28:57.491653525+08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2025-07-04T22:06:52.200518707+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": unexpected EOF"}
+{"time":"2025-07-04T22:09:45.338273816+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": unexpected EOF"}
+{"time":"2025-07-04T22:19:20.574743081+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:59078->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-04T22:25:54.288016702+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:42132->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-04T22:29:40.591991523+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:37544->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-04T22:36:54.256091094+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:40968->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-04T22:37:22.364944108+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": unexpected EOF"}
+{"time":"2025-07-04T22:40:24.499117928+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-07-04T22:40:51.249223858+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": unexpected EOF"}
+{"time":"2025-07-04T22:44:05.872015851+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:59778->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-04T22:49:18.192032141+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:47358->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-07-04T22:52:13.295997002+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:46182->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-04T22:53:26.345699486+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": unexpected EOF"}
+{"time":"2025-07-04T22:55:37.691524069+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": unexpected EOF"}
+{"time":"2025-07-04T22:59:01.477384402+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": unexpected EOF"}
+{"time":"2025-07-04T23:01:22.224282887+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:45222->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-07-04T23:06:44.720013857+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:46280->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-07-04T23:08:42.894770628+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-07-04T23:10:13.616061547+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:57034->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-04T23:11:27.896127402+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-07-04T23:15:47.19805854+08:00","level":"ERROR","msg":"filestream: json decode error: net/http: request canceled (Client.Timeout or context cancellation while reading body)"}
+{"time":"2025-07-04T23:15:47.222866077+08:00","level":"ERROR","msg":"filestream: error closing response body: net/http: request canceled"}
+{"time":"2025-07-04T23:19:26.063989295+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:35504->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-04T23:21:57.905369451+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-07-04T23:23:00.080992848+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:38378->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-04T23:26:54.577250259+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:41928->104.21.20.172:443: read: connection reset by peer"}
+{"time":"2025-07-04T23:28:47.703904029+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:38040->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-07-04T23:30:12.910139882+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-07-04T23:30:45.313312591+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-07-04T23:35:25.039973358+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:38402->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-07-04T23:39:27.49206097+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:39926->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-07-04T23:43:09.424012888+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:59448->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-04T23:46:07.600020006+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:33610->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-04T23:46:28.951111977+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": unexpected EOF"}
+{"time":"2025-07-04T23:48:12.919414088+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-07-04T23:48:45.403207458+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-07-04T23:49:16.527984782+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:49472->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-04T23:50:27.921623046+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-07-04T23:52:57.899934024+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:40636->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-07-04T23:56:18.928962652+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:53056->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-04T23:56:57.924908638+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-07-04T23:57:30.375318804+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-07-04T23:58:04.586629939+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-07-05T00:03:09.552010125+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:39630->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-05T00:03:42.930344983+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-07-05T00:04:15.375941679+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-07-05T00:04:21.655905995+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:48150->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-07-05T00:04:50.22664016+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-07-05T00:08:01.391966638+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:53260->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-07-05T00:12:23.023992865+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:44698->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-05T00:15:42.93731147+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-07-05T00:16:15.214992648+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-07-05T00:16:49.667525584+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-07-05T00:17:01.040050871+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:42760->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-05T00:20:02.288062562+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:37934->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-07-05T00:23:13.264033499+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:51502->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-05T00:26:32.944001316+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:50460->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-05T00:30:17.199971228+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:43756->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-07-05T00:35:42.94748626+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-07-05T00:35:44.881161178+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:59610->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-05T00:39:14.287974585+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:46688->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-07-05T00:39:33.427697791+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:40430->104.21.20.172:443: read: connection reset by peer"}
+{"time":"2025-07-05T00:40:47.648388331+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:60164->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-07-05T00:42:34.088456552+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:52978->104.21.20.172:443: read: connection reset by peer"}
+{"time":"2025-07-05T00:47:13.006425282+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:33354->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-07-05T00:49:55.823998082+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:33330->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-05T00:52:51.439993456+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:52416->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-05T00:57:07.440983899+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:39188->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-05T01:04:51.312039238+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:59436->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-05T01:09:40.080000713+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:34348->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-05T01:18:31.535996696+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:57184->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-07-05T01:25:12.431983593+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_07041727_2dataset/6bkqzmou/file_stream\": read tcp 10.1.9.185:59852->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-05T11:36:49.210079644+08:00","level":"INFO","msg":"stream: closing","id":"6bkqzmou"}
+{"time":"2025-07-05T11:36:49.210163239+08:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2025-07-05T11:36:49.211103046+08:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2025-07-05T11:36:51.804545543+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-07-05T11:36:53.755788884+08:00","level":"INFO","msg":"handler: closed","stream_id":"6bkqzmou"}
+{"time":"2025-07-05T11:36:53.755828602+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"6bkqzmou"}
+{"time":"2025-07-05T11:36:53.75584333+08:00","level":"INFO","msg":"sender: closed","stream_id":"6bkqzmou"}
+{"time":"2025-07-05T11:36:53.759902053+08:00","level":"INFO","msg":"stream: closed","id":"6bkqzmou"}

ProtT3/all_checkpoints/stage1_07041727_2dataset/wandb/run-20250704_172854-6bkqzmou/logs/debug.log

@@ -0,0 +1,24 @@
+2025-07-04 17:28:55,100 INFO    MainThread:29356 [wandb_setup.py:_flush():70] Current SDK version is 0.19.11
+2025-07-04 17:28:55,100 INFO    MainThread:29356 [wandb_setup.py:_flush():70] Configure stats pid to 29356
+2025-07-04 17:28:55,100 INFO    MainThread:29356 [wandb_setup.py:_flush():70] Loading settings from /root/.config/wandb/settings
+2025-07-04 17:28:55,100 INFO    MainThread:29356 [wandb_setup.py:_flush():70] Loading settings from /nas/shared/kilab/wangyujia/ProtT3/wandb/settings
+2025-07-04 17:28:55,100 INFO    MainThread:29356 [wandb_setup.py:_flush():70] Loading settings from environment variables
+2025-07-04 17:28:55,100 INFO    MainThread:29356 [wandb_init.py:setup_run_log_directory():724] Logging user logs to ./all_checkpoints/stage1_07041727_2dataset/wandb/run-20250704_172854-6bkqzmou/logs/debug.log
+2025-07-04 17:28:55,100 INFO    MainThread:29356 [wandb_init.py:setup_run_log_directory():725] Logging internal logs to ./all_checkpoints/stage1_07041727_2dataset/wandb/run-20250704_172854-6bkqzmou/logs/debug-internal.log
+2025-07-04 17:28:55,100 INFO    MainThread:29356 [wandb_init.py:init():852] calling init triggers
+2025-07-04 17:28:55,100 INFO    MainThread:29356 [wandb_init.py:init():857] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-07-04 17:28:55,100 INFO    MainThread:29356 [wandb_init.py:init():893] starting backend
+2025-07-04 17:28:55,100 INFO    MainThread:29356 [wandb_init.py:init():897] sending inform_init request
+2025-07-04 17:28:55,102 INFO    MainThread:29356 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-07-04 17:28:55,102 INFO    MainThread:29356 [wandb_init.py:init():907] backend started and connected
+2025-07-04 17:28:55,103 INFO    MainThread:29356 [wandb_init.py:init():1005] updated telemetry
+2025-07-04 17:28:55,103 INFO    MainThread:29356 [wandb_init.py:init():1029] communicating run to backend with 90.0 second timeout
+2025-07-04 17:28:57,453 INFO    MainThread:29356 [wandb_init.py:init():1104] starting run threads in backend
+2025-07-04 17:28:57,668 INFO    MainThread:29356 [wandb_run.py:_console_start():2573] atexit reg
+2025-07-04 17:28:57,669 INFO    MainThread:29356 [wandb_run.py:_redirect():2421] redirect: wrap_raw
+2025-07-04 17:28:57,678 INFO    MainThread:29356 [wandb_run.py:_redirect():2490] Wrapping output streams.
+2025-07-04 17:28:57,684 INFO    MainThread:29356 [wandb_run.py:_redirect():2513] Redirects installed.
+2025-07-04 17:28:57,686 INFO    MainThread:29356 [wandb_init.py:init():1150] run started, returning control to user process
+2025-07-04 17:29:03,015 INFO    MainThread:29356 [wandb_run.py:_config_callback():1436] config_cb None None {'filename': 'stage1_07041727_2dataset', 'seed': 42, 'mode': 'train', 'strategy': 'deepspeed', 'accelerator': 'gpu', 'devices': '0,1,2,3,4,5,6,7', 'precision': 'bf16-mixed', 'max_epochs': 30, 'check_val_every_n_epoch': 1, 'use_wandb_logger': True, 'mix_dataset': True, 'temperature': 0.1, 'save_every_n_epochs': 5, 'ptm': True, 'lm': True, 'rerank_cand_num': 128, 'plm_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m', 'plm_tune': 'freeze', 'load_4bit': False, 'pool_size': 0, 'bert_hidden_dim': 768, 'bert_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft', 'projection_dim': 256, 'cross_attention_freq': 2, 'num_query_token': 8, 'weight_decay': 0.05, 'init_lr': 0.0001, 'min_lr': 1e-05, 'warmup_lr': 1e-06, 'warmup_steps': 1000, 'lr_decay_rate': 0.9, 'scheduler': 'linear_warmup_cosine_lr', 'init_checkpoint': '', 'retrieval_eval_epoch': 10, 'num_workers': 8, 'batch_size': 32, 'match_batch_size': 64, 'root': 'data', 'text_max_len': 128, 'prot_max_len': 1024, 'prot_aug': 'None'}
+2025-07-05 11:36:49,208 INFO    MsgRouterThr:29356 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles.

ProtT3/all_checkpoints/stage1_ckpt/wandb/debug-internal.log

@@ -0,0 +1,19 @@
+{"time":"2025-06-28T22:23:55.245053432+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.11","symlink path":"all_checkpoints/stage1_ckpt/wandb/run-20250628_222355-e9wtzwz1/logs/debug-core.log"}
+{"time":"2025-06-28T22:23:56.442748363+08:00","level":"INFO","msg":"created new stream","id":"e9wtzwz1"}
+{"time":"2025-06-28T22:23:56.442786823+08:00","level":"INFO","msg":"stream: started","id":"e9wtzwz1"}
+{"time":"2025-06-28T22:23:56.44283909+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"e9wtzwz1"}
+{"time":"2025-06-28T22:23:56.44287811+08:00","level":"INFO","msg":"sender: started","stream_id":"e9wtzwz1"}
+{"time":"2025-06-28T22:23:56.442850569+08:00","level":"INFO","msg":"handler: started","stream_id":"e9wtzwz1"}
+{"time":"2025-06-28T22:23:57.657067842+08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2025-06-28T23:13:08.786733475+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_ckpt/e9wtzwz1/file_stream\": read tcp 10.1.6.17:41088->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-06-28T23:16:23.858735046+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_ckpt/e9wtzwz1/file_stream\": read tcp 10.1.6.17:58168->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-28T23:20:12.333412842+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_ckpt/e9wtzwz1/file_stream\": read tcp 10.1.6.17:50886->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-06-28T23:28:29.895934993+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_ckpt/e9wtzwz1/file_stream\": unexpected EOF"}
+{"time":"2025-06-28T23:32:39.731699923+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_ckpt/e9wtzwz1/file_stream\": read tcp 10.1.6.17:54496->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-28T23:35:17.938724051+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_ckpt/e9wtzwz1/file_stream\": read tcp 10.1.6.17:33406->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-06-28T23:38:54.515701632+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_ckpt/e9wtzwz1/file_stream\": read tcp 10.1.6.17:59930->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-06-28T23:41:28.010949965+08:00","level":"INFO","msg":"stream: closing","id":"e9wtzwz1"}
+{"time":"2025-06-28T23:41:28.011132748+08:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2025-06-28T23:41:28.066664522+08:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2025-06-28T23:41:37.996261564+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-06-28T23:42:36.21077519+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_ckpt/e9wtzwz1/file_stream\": read tcp 10.1.6.17:36806->172.67.193.61:443: read: connection timed out"}

ProtT3/all_checkpoints/stage1_ckpt/wandb/debug.log

@@ -0,0 +1,23 @@
+2025-06-28 22:23:55,235 INFO    MainThread:3589 [wandb_setup.py:_flush():70] Current SDK version is 0.19.11
+2025-06-28 22:23:55,235 INFO    MainThread:3589 [wandb_setup.py:_flush():70] Configure stats pid to 3589
+2025-06-28 22:23:55,235 INFO    MainThread:3589 [wandb_setup.py:_flush():70] Loading settings from /root/.config/wandb/settings
+2025-06-28 22:23:55,235 INFO    MainThread:3589 [wandb_setup.py:_flush():70] Loading settings from /nas/shared/kilab/wangyujia/ProtT3/wandb/settings
+2025-06-28 22:23:55,235 INFO    MainThread:3589 [wandb_setup.py:_flush():70] Loading settings from environment variables
+2025-06-28 22:23:55,235 INFO    MainThread:3589 [wandb_init.py:setup_run_log_directory():724] Logging user logs to ./all_checkpoints/stage1_ckpt/wandb/run-20250628_222355-e9wtzwz1/logs/debug.log
+2025-06-28 22:23:55,235 INFO    MainThread:3589 [wandb_init.py:setup_run_log_directory():725] Logging internal logs to ./all_checkpoints/stage1_ckpt/wandb/run-20250628_222355-e9wtzwz1/logs/debug-internal.log
+2025-06-28 22:23:55,235 INFO    MainThread:3589 [wandb_init.py:init():852] calling init triggers
+2025-06-28 22:23:55,235 INFO    MainThread:3589 [wandb_init.py:init():857] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-06-28 22:23:55,235 INFO    MainThread:3589 [wandb_init.py:init():893] starting backend
+2025-06-28 22:23:55,235 INFO    MainThread:3589 [wandb_init.py:init():897] sending inform_init request
+2025-06-28 22:23:55,237 INFO    MainThread:3589 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-06-28 22:23:55,239 INFO    MainThread:3589 [wandb_init.py:init():907] backend started and connected
+2025-06-28 22:23:55,240 INFO    MainThread:3589 [wandb_init.py:init():1005] updated telemetry
+2025-06-28 22:23:55,249 INFO    MainThread:3589 [wandb_init.py:init():1029] communicating run to backend with 90.0 second timeout
+2025-06-28 22:23:57,648 INFO    MainThread:3589 [wandb_init.py:init():1104] starting run threads in backend
+2025-06-28 22:23:57,823 INFO    MainThread:3589 [wandb_run.py:_console_start():2573] atexit reg
+2025-06-28 22:23:57,823 INFO    MainThread:3589 [wandb_run.py:_redirect():2421] redirect: wrap_raw
+2025-06-28 22:23:57,827 INFO    MainThread:3589 [wandb_run.py:_redirect():2490] Wrapping output streams.
+2025-06-28 22:23:57,827 INFO    MainThread:3589 [wandb_run.py:_redirect():2513] Redirects installed.
+2025-06-28 22:23:57,829 INFO    MainThread:3589 [wandb_init.py:init():1150] run started, returning control to user process
+2025-06-28 23:41:27,993 INFO    MsgRouterThr:3589 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 2 handles.

ProtT3/all_checkpoints/stage1_ckpt/wandb/run-20250628_205450-irx8yzsh/files/config.yaml

@@ -0,0 +1,129 @@
+_wandb:
+    value:
+        cli_version: 0.19.11
+        m:
+            - "1": trainer/global_step
+              "6":
+                - 3
+              "7": []
+        python_version: 3.10.0
+        t:
+            "1":
+                - 1
+                - 5
+                - 9
+                - 11
+                - 33
+                - 41
+                - 49
+                - 53
+                - 55
+                - 63
+                - 103
+            "2":
+                - 1
+                - 5
+                - 9
+                - 11
+                - 33
+                - 41
+                - 49
+                - 53
+                - 55
+                - 63
+                - 103
+            "3":
+                - 7
+                - 23
+                - 33
+                - 55
+                - 66
+            "4": 3.10.0
+            "5": 0.19.11
+            "6": 4.52.3
+            "8":
+                - 5
+            "12": 0.19.11
+            "13": linux-x86_64
+accelerator:
+    value: gpu
+batch_size:
+    value: 32
+bert_hidden_dim:
+    value: 768
+bert_name:
+    value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft
+check_val_every_n_epoch:
+    value: 1
+cross_attention_freq:
+    value: 2
+devices:
+    value: 0,1,2,3,4,5,6,7
+filename:
+    value: stage1_ckpt
+init_checkpoint:
+    value: ""
+init_lr:
+    value: 0.0001
+lm:
+    value: true
+load_4bit:
+    value: false
+lr_decay_rate:
+    value: 0.9
+match_batch_size:
+    value: 64
+max_epochs:
+    value: 20
+min_lr:
+    value: 1e-05
+mix_dataset:
+    value: true
+mode:
+    value: train
+num_query_token:
+    value: 8
+num_workers:
+    value: 8
+plm_name:
+    value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m
+plm_tune:
+    value: freeze
+pool_size:
+    value: 0
+precision:
+    value: bf16-mixed
+projection_dim:
+    value: 256
+prot_aug:
+    value: None
+prot_max_len:
+    value: 1024
+ptm:
+    value: true
+rerank_cand_num:
+    value: 128
+retrieval_eval_epoch:
+    value: 10
+root:
+    value: data_small
+save_every_n_epochs:
+    value: 5
+scheduler:
+    value: linear_warmup_cosine_lr
+seed:
+    value: 42
+strategy:
+    value: ddp
+temperature:
+    value: 0.1
+text_max_len:
+    value: 128
+use_wandb_logger:
+    value: true
+warmup_lr:
+    value: 1e-06
+warmup_steps:
+    value: 1000
+weight_decay:
+    value: 0.05

ProtT3/all_checkpoints/stage1_ckpt/wandb/run-20250628_205450-irx8yzsh/files/output.log

@@ -0,0 +1,2 @@
+
+Detected KeyboardInterrupt, attempting graceful shutdown ...

ProtT3/all_checkpoints/stage1_ckpt/wandb/run-20250628_205450-irx8yzsh/files/requirements.txt

@@ -0,0 +1,225 @@
+marisa-trie==1.2.1
+pydantic==2.11.5
+mdurl==0.1.2
+gitdb==4.0.12
+scikit-image==0.25.2
+async-timeout==5.0.1
+blis==1.3.0
+urllib3==2.4.0
+spacy==3.8.7
+nvidia-ml-py==12.575.51
+braceexpand==0.1.7
+nvidia-cufft-cu12==11.2.1.3
+rich==14.0.0
+setuptools==78.1.1
+matplotlib==3.10.3
+catalogue==2.0.10
+decord==0.6.0
+numpy==2.2.6
+charset-normalizer==3.4.2
+langcodes==3.5.0
+pexpect==4.9.0
+nltk==3.9.1
+cachetools==5.5.2
+cfgv==3.4.0
+prompt_toolkit==3.0.51
+srsly==2.5.1
+einops==0.8.1
+Jinja2==3.1.6
+cloudpathlib==0.21.1
+streamlit==1.45.1
+pydantic_core==2.33.2
+tornado==6.5.1
+nvidia-curand-cu12==10.3.5.147
+deepspeed==0.16.10+b666844f
+networkx==3.4.2
+omegaconf==2.3.0
+msgpack==1.1.0
+pandas==2.2.3
+rouge_score==0.1.2
+six==1.17.0
+language_data==1.3.0
+referencing==0.36.2
+rpds-py==0.25.1
+lazy_loader==0.4
+pydeck==0.9.1
+markdown-it-py==3.0.0
+fonttools==4.58.0
+nvidia-cuda-runtime-cu12==12.4.127
+smart-open==7.1.0
+identify==2.6.12
+pure_eval==0.2.3
+confection==0.1.5
+nvidia-cublas-cu12==12.4.5.8
+nvidia-cusparselt-cu12==0.6.2
+decorator==5.2.1
+nvidia-nccl-cu12==2.21.5
+pytz==2025.2
+nvidia-cudnn-cu12==9.1.0.70
+plotly==6.1.1
+safetensors==0.5.3
+portalocker==3.1.1
+toml==0.10.2
+triton==3.2.0
+cycler==0.12.1
+torch==2.6.0
+python-magic==0.4.27
+ptyprocess==0.7.0
+regex==2024.11.6
+absl-py==2.2.2
+psutil==7.0.0
+murmurhash==1.0.13
+wrapt==1.17.2
+pycocoevalcap==1.2
+python-slugify==8.0.4
+stack-data==0.6.3
+python-dateutil==2.9.0.post0
+scipy==1.15.3
+annotated-types==0.7.0
+mpmath==1.3.0
+ipython==8.36.0
+pyparsing==3.2.3
+nvidia-nvtx-cu12==12.4.127
+fairscale==0.4.4
+jsonschema-specifications==2025.4.1
+matplotlib-inline==0.1.7
+watchdog==6.0.0
+thinc==8.3.6
+antlr4-python3-runtime==4.9.3
+webencodings==0.5.1
+hjson==3.1.0
+propcache==0.3.1
+virtualenv==20.31.2
+pytorch-lightning==2.5.1.post0
+Pygments==2.19.1
+pillow==11.2.1
+joblib==1.5.1
+tqdm==4.67.1
+timm==0.4.12
+nvidia-nvjitlink-cu12==12.4.127
+aiosignal==1.3.2
+kaggle==1.7.4.5
+idna==3.10
+pycocotools==2.0.8
+MarkupSafe==3.0.2
+traitlets==5.14.3
+multidict==6.4.4
+distlib==0.3.9
+torchmetrics==1.7.1
+pyarrow==20.0.0
+tzdata==2025.2
+platformdirs==4.3.8
+yarl==1.20.0
+tenacity==9.1.2
+altair==5.5.0
+wasabi==1.1.3
+attrs==25.3.0
+contourpy==1.3.2
+kiwisolver==1.4.8
+PyYAML==6.0.2
+exceptiongroup==1.3.0
+jedi==0.19.2
+sentencepiece==0.2.0
+nvidia-cusolver-cu12==11.6.1.9
+requests==2.32.3
+opendatasets==0.1.22
+GitPython==3.1.44
+bleach==6.2.0
+protobuf==6.31.0
+sympy==1.13.1
+filelock==3.18.0
+pre_commit==4.2.0
+text-unidecode==1.3
+wheel==0.45.1
+contexttimer==0.3.3
+wcwidth==0.2.13
+spacy-legacy==3.0.12
+aiohappyeyeballs==2.6.1
+imageio==2.37.0
+nodeenv==1.9.1
+py-cpuinfo==9.0.0
+hf-xet==1.1.2
+nvidia-cuda-cupti-cu12==12.4.127
+weasel==0.4.1
+certifi==2025.4.26
+lightning-utilities==0.14.3
+typing_extensions==4.13.2
+typing-inspection==0.4.1
+webdataset==0.2.111
+nvidia-cusparse-cu12==12.3.1.170
+asttokens==3.0.0
+nvidia-cufile-cu12==1.11.1.6
+opencv-python-headless==4.5.5.64
+smmap==5.0.2
+tifffile==2025.5.10
+iopath==0.1.10
+packaging==24.2
+cymem==2.0.11
+spacy-loggers==1.0.5
+ninja==1.11.1.4
+ftfy==6.3.1
+executing==2.2.0
+nvidia-cuda-nvrtc-cu12==12.4.127
+blinker==1.9.0
+torchvision==0.21.0
+parso==0.8.4
+salesforce-lavis==1.0.2
+frozenlist==1.6.0
+shellingham==1.5.4
+flash-attn==2.7.1.post1
+pycparser==2.22
+threadpoolctl==3.6.0
+opencv-python==4.11.0.86
+fsspec==2025.3.0
+aiohttp==3.12.2
+narwhals==1.41.0
+opendelta==0.3.2
+pycryptodome==3.23.0
+crcmod==1.7
+delta-center-client==0.0.4
+tokenizers==0.21.1
+aliyun-python-sdk-kms==2.16.5
+more-itertools==10.7.0
+yacs==0.1.8
+bigmodelvis==0.0.1
+jmespath==0.10.0
+docker-pycreds==0.4.0
+web.py==0.62
+scikit-learn==1.6.1
+pip==25.1.1
+cheroot==10.0.1
+setproctitle==1.3.6
+huggingface-hub==0.32.1
+oss2==2.15.0
+cryptography==45.0.3
+typer==0.16.0
+xxhash==3.5.0
+jsonschema==4.24.0
+click==8.2.1
+preshed==3.0.10
+sentry-sdk==2.29.1
+wandb==0.19.11
+dill==0.3.8
+aliyun-python-sdk-core==2.16.0
+transformers==4.52.3
+cffi==1.17.1
+pathlib==1.0.1
+jaraco.functools==4.1.0
+datasets==3.6.0
+multiprocess==0.70.16
+backports.tarfile==1.2.0
+tomli==2.0.1
+autocommand==2.2.2
+zipp==3.19.2
+jaraco.text==3.12.1
+jaraco.collections==5.1.0
+platformdirs==4.2.2
+typeguard==4.3.0
+typing_extensions==4.12.2
+jaraco.functools==4.0.1
+inflect==7.3.1
+wheel==0.45.1
+more-itertools==10.3.0
+importlib_metadata==8.0.0
+jaraco.context==5.3.0
+packaging==24.2