Add files using upload-large-folder tool

BioReason_new/wandb/run-20250813_121906-3wcb4wer/files/requirements.txt

@@ -0,0 +1,233 @@
+jupyterlab_pygments==0.3.0
+depyf==0.19.0
+av==15.0.0
+fastapi==0.116.1
+ray==2.48.0
+cachetools==6.1.0
+prometheus_client==0.22.1
+requests==2.32.4
+jupyter-lsp==2.2.6
+jupyter-events==0.12.0
+anyio==4.10.0
+pybase64==1.4.2
+fastjsonschema==2.21.1
+cloudpickle==3.1.1
+sentry-sdk==2.34.1
+torchmetrics==1.8.1
+wheel==0.45.1
+idna==3.10
+defusedxml==0.7.1
+starlette==0.47.2
+python-dotenv==1.1.1
+comm==0.2.3
+multidict==6.6.3
+python-json-logger==3.3.0
+jupyter_server==2.16.0
+nvidia-nvtx-cu12==12.6.77
+websockets==15.0.1
+referencing==0.36.2
+gguf==0.17.1
+lm-format-enforcer==0.10.12
+xformers==0.0.31
+ninja==1.11.1.4
+beautifulsoup4==4.13.4
+dnspython==2.7.0
+certifi==2025.8.3
+markdown-it-py==3.0.0
+stack-data==0.6.3
+nvidia-cusparselt-cu12==0.6.3
+networkx==3.5
+aiohappyeyeballs==2.6.1
+nvidia-cuda-cupti-cu12==12.6.80
+llguidance==0.7.30
+rich==14.1.0
+outlines_core==0.2.10
+nvidia-nccl-cu12==2.26.2
+pandas==2.3.1
+pydantic_core==2.33.2
+argon2-cffi==25.1.0
+babel==2.17.0
+isoduration==20.11.0
+uvloop==0.21.0
+json5==0.12.0
+widgetsnbextension==4.0.14
+aiohttp==3.12.15
+notebook_shim==0.2.4
+pandocfilters==1.5.1
+vllm==0.10.0
+blake3==1.0.5
+diskcache==5.6.3
+h11==0.16.0
+attrs==25.3.0
+nvidia-cusolver-cu12==11.7.1.2
+click==8.2.1
+cupy-cuda12x==13.5.1
+jedi==0.19.2
+py-cpuinfo==9.0.0
+overrides==7.7.0
+pytz==2025.2
+protobuf==6.31.1
+dill==0.3.8
+ipykernel==6.30.1
+frozenlist==1.7.0
+huggingface-hub==0.34.4
+rignore==0.6.4
+mistune==3.1.3
+sniffio==1.3.1
+httpx==0.28.1
+sentencepiece==0.2.0
+typing_extensions==4.14.1
+typing-inspection==0.4.1
+async-lru==2.0.5
+bioreason==0.1.0
+charset-normalizer==3.4.3
+llvmlite==0.44.0
+safetensors==0.6.2
+fqdn==1.5.1
+peft==0.17.0
+fastapi-cli==0.0.8
+torch==2.7.1
+jsonschema==4.25.0
+torchvision==0.22.1
+parso==0.8.4
+pillow==11.3.0
+pycountry==24.6.1
+scipy==1.16.1
+wandb==0.21.1
+pip==25.1
+wcwidth==0.2.13
+mistral_common==1.8.3
+nvidia-cudnn-cu12==9.5.1.17
+asttokens==3.0.0
+numpy==2.2.6
+uvicorn==0.35.0
+jupyter_client==8.6.3
+debugpy==1.8.16
+fastapi-cloud-cli==0.1.5
+compressed-tensors==0.10.2
+GitPython==3.1.45
+nvidia-ml-py==13.580.65
+tiktoken==0.11.0
+partial-json-parser==0.2.1.1.post6
+prompt_toolkit==3.0.51
+uri-template==1.3.0
+pyarrow==21.0.0
+nvidia-cuda-nvrtc-cu12==12.6.77
+propcache==0.3.2
+matplotlib-inline==0.1.7
+pure_eval==0.2.3
+nest-asyncio==1.6.0
+deepspeed==0.17.4
+einops==0.8.1
+triton==3.3.1
+trl==0.21.0
+pyzmq==27.0.1
+httptools==0.6.4
+ipython==9.4.0
+packaging==25.0
+hjson==3.1.0
+soxr==0.5.0.post1
+webcolors==24.11.1
+PyYAML==6.0.2
+ipywidgets==8.1.7
+lark==1.2.2
+python-multipart==0.0.20
+tinycss2==1.4.0
+mpmath==1.3.0
+psutil==7.0.0
+nbformat==5.10.4
+qwen-vl-utils==0.0.11
+notebook==7.4.5
+jupyterlab==4.4.5
+transformers==4.55.0
+urllib3==2.5.0
+terminado==0.18.1
+xgrammar==0.1.21
+xxhash==3.5.0
+tzdata==2025.2
+jupyterlab_widgets==3.0.15
+aiosignal==1.4.0
+annotated-types==0.7.0
+nvidia-nvjitlink-cu12==12.6.85
+httpcore==1.0.9
+pexpect==4.9.0
+tornado==6.5.2
+regex==2025.7.33
+filelock==3.18.0
+nvidia-cuda-runtime-cu12==12.6.77
+openai==1.90.0
+Pygments==2.19.2
+cffi==1.17.1
+nvidia-cufft-cu12==11.3.0.4
+rpds-py==0.27.0
+rich-toolkit==0.15.0
+bleach==6.2.0
+sympy==1.14.0
+python-dateutil==2.9.0.post0
+watchfiles==1.1.0
+nbclient==0.10.2
+hf-xet==1.1.7
+Send2Trash==1.8.3
+smmap==5.0.2
+tokenizers==0.21.4
+yarl==1.20.1
+MarkupSafe==3.0.2
+prometheus-fastapi-instrumentator==7.1.0
+rfc3986-validator==0.1.1
+bitsandbytes==0.46.1
+nvidia-cufile-cu12==1.11.1.6
+fsspec==2025.3.0
+executing==2.2.0
+pydantic==2.11.7
+jupyter_core==5.8.1
+msgspec==0.19.0
+nvidia-cusparse-cu12==12.5.4.2
+opencv-python-headless==4.12.0.88
+webencodings==0.5.1
+ptyprocess==0.7.0
+pytorch-lightning==2.5.2
+Jinja2==3.1.6
+websocket-client==1.8.0
+jsonpointer==3.0.0
+traitlets==5.14.3
+numba==0.61.2
+soupsieve==2.7
+cbor2==5.6.5
+multiprocess==0.70.16
+email_validator==2.2.0
+distro==1.9.0
+accelerate==1.10.0
+torchaudio==2.7.1
+jiter==0.10.0
+rfc3339-validator==0.1.4
+jupyter==1.1.1
+typer==0.16.0
+gitdb==4.0.12
+types-python-dateutil==2.9.0.20250809
+soundfile==0.13.1
+astor==0.8.1
+shellingham==1.5.4
+jupyterlab_server==2.27.3
+nvidia-curand-cu12==10.3.7.77
+jupyter-console==6.6.3
+msgpack==1.1.1
+six==1.17.0
+pycparser==2.22
+arrow==1.3.0
+tqdm==4.67.1
+lightning-utilities==0.15.2
+interegular==0.3.3
+rfc3987-syntax==1.1.0
+ipython_pygments_lexers==1.1.1
+datasets==4.0.0
+jupyter_server_terminals==0.5.3
+platformdirs==4.3.8
+pydantic-extra-types==2.10.5
+decorator==5.2.1
+mdurl==0.1.2
+nvidia-cublas-cu12==12.6.4.1
+jsonschema-specifications==2025.4.1
+setuptools==78.1.1
+fastrlock==0.8.3
+argon2-cffi-bindings==25.1.0
+nbconvert==7.16.6

BioReason_new/wandb/run-20250813_121906-3wcb4wer/files/wandb-metadata.json

@@ -0,0 +1,116 @@
+{
+  "os": "Linux-5.10.134-008.18.kangaroo.al8.x86_64-x86_64-with-glibc2.35",
+  "python": "CPython 3.11.0",
+  "startedAt": "2025-08-13T04:19:06.448075Z",
+  "args": [
+    "--text_model_name",
+    "/oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged",
+    "--protein_model_name",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m",
+    "--qformer_model_name",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft",
+    "--num_query_tokens",
+    "8",
+    "--train_dataset",
+    "/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl",
+    "--valid_dataset",
+    "/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl",
+    "--output_dir",
+    "./contrastive_outputs",
+    "--num_epochs",
+    "10",
+    "--batch_size",
+    "32",
+    "--learning_rate",
+    "1e-4",
+    "--temperature",
+    "0.07",
+    "--freeze_protein_model",
+    "--freeze_text_model",
+    "--enable_ptm",
+    "--max_length_protein",
+    "1024",
+    "--max_length_text",
+    "512",
+    "--num_workers",
+    "8",
+    "--eval_dataset",
+    "--use_wandb",
+    "--wandb_project",
+    "protein-llm-contrastive",
+    "--logging_steps",
+    "100",
+    "--eval_steps",
+    "500",
+    "--save_steps",
+    "1000"
+  ],
+  "program": "/nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py",
+  "codePath": "wangyujia/BioReason_new/train_contrastive.py",
+  "codePathLocal": "train_contrastive.py",
+  "git": {
+    "remote": "https://github.com/PorUna-byte/PAR.git",
+    "commit": "b8caf406aa1699c788f0ca6e44a1769452c317db"
+  },
+  "email": "gia0603yucca@gmail.com",
+  "root": "/nas/shared/kilab/wangyujia/BioReason_new",
+  "host": "dsw-265304-585cc9d768-ckfd7",
+  "executable": "/root/miniconda3/envs/bioreason/bin/python",
+  "cpu_count": 64,
+  "cpu_count_logical": 64,
+  "gpu": "NVIDIA A800-SXM4-80GB",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1623302262784",
+      "used": "1014808576"
+    }
+  },
+  "memory": {
+    "total": "549755813888"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-62fcc532-880f-7324-eb13-3b739a38c29b"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-af1197aa-3e7d-a53e-185d-04b1d64164da"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-3e6e551f-2d7d-a27b-654a-da56933d2283"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-a947b64a-c545-ee5e-30e6-b4b0e2dd8fdc"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-ebd04f3f-c437-1b8b-26f2-4b1ebfee93c4"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-53994891-72a6-25be-a4c8-e32c695155b0"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-a927d00a-15c5-31f5-9ae1-9d2e773821a1"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-93d555c4-29a0-cd78-abac-d98cd160d4e1"
+    }
+  ],
+  "cudaVersion": "12.1",
+  "writerId": "eku61gd0lj70gwdugjle81nramcib5o4"
+}

BioReason_new/wandb/run-20250813_121906-3wcb4wer/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":1910},"_runtime":1910}

BioReason_new/wandb/run-20250813_121906-3wcb4wer/logs/debug-internal.log

@@ -0,0 +1,18 @@
+{"time":"2025-08-13T12:19:06.822910203+08:00","level":"INFO","msg":"stream: starting","core version":"0.21.1"}
+{"time":"2025-08-13T12:19:08.068655385+08:00","level":"INFO","msg":"stream: created new stream","id":"3wcb4wer"}
+{"time":"2025-08-13T12:19:08.068708947+08:00","level":"INFO","msg":"stream: started","id":"3wcb4wer"}
+{"time":"2025-08-13T12:19:08.068747833+08:00","level":"INFO","msg":"writer: started","stream_id":"3wcb4wer"}
+{"time":"2025-08-13T12:19:08.068766174+08:00","level":"INFO","msg":"sender: started","stream_id":"3wcb4wer"}
+{"time":"2025-08-13T12:19:08.06880027+08:00","level":"INFO","msg":"handler: started","stream_id":"3wcb4wer"}
+{"time":"2025-08-13T12:19:09.937699234+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/3wcb4wer/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T041909Z&X-Goog-Expires=86399&X-Goog-Signature=2235d1f23ffa94e46560d8d9f1d99a989e9a6742f86a4808f9aeb398930bff09a6648b165f02c86e3336b2c45dea184c1c00bfc510a911f21d1d3c93af8315df007c4be4d48a40d509eddf912bad9dad6111e122f0c52e4718f52e9920b371ccf7d661507efed6a4dca98d6905e4aa52888a2f967d3b5cb0a7edb86ae58b67aa789d6b6d0baeadacdb94ddddd35daa1abcc27411f73ce268720d2a07f37be8e791e57cb72538e70626bd1fa9f490e39a22eeeb4e0ba62fa31b4209ee026d51d1e750551a6dcbfd81ee69fd25b7a1377e4688b46f170feafeb3a758aa1d9e1b881ee4009e8798452e7af98f262897f880de73f172a6426c917978ef0925e2e600&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:51032->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/3wcb4wer/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T041909Z&X-Goog-Expires=86399&X-Goog-Signature=2235d1f23ffa94e46560d8d9f1d99a989e9a6742f86a4808f9aeb398930bff09a6648b165f02c86e3336b2c45dea184c1c00bfc510a911f21d1d3c93af8315df007c4be4d48a40d509eddf912bad9dad6111e122f0c52e4718f52e9920b371ccf7d661507efed6a4dca98d6905e4aa52888a2f967d3b5cb0a7edb86ae58b67aa789d6b6d0baeadacdb94ddddd35daa1abcc27411f73ce268720d2a07f37be8e791e57cb72538e70626bd1fa9f490e39a22eeeb4e0ba62fa31b4209ee026d51d1e750551a6dcbfd81ee69fd25b7a1377e4688b46f170feafeb3a758aa1d9e1b881ee4009e8798452e7af98f262897f880de73f172a6426c917978ef0925e2e600&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T12:21:20.029385617+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/3wcb4wer/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T041909Z&X-Goog-Expires=86399&X-Goog-Signature=810610a47880236a95ccb7704d4c0dc16f7a0b7eae65667872dce5f537b288a571822f41e7a45f3e344f5d667ba7e46b08bfe38190c6996e05cc6271d589340f91b659b04ad279a66aaf29fc5c234107aabf0c722489b48ea64d9be719128f32bc5d56b7b2f19e2bb2e0134714913c7279b194400e5b22f24cbaa2b825742be756fbb257747c6677c2e141f437e9bb64d7ff4c573fde01df4f40d20a5ba4f23853dc33d89cdfc237721a16212ba8ad2d8d30430713ee477b5187b4ccb50edd3683597d5a99258908fe9e74178a8be41e73ab77fe17889dcd7542db1f8ceaa8c5fe918adf111dfdd605f40c19e93f9ad543d872111ec416aa3cb5e60909c01345&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:36830->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/3wcb4wer/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T041909Z&X-Goog-Expires=86399&X-Goog-Signature=810610a47880236a95ccb7704d4c0dc16f7a0b7eae65667872dce5f537b288a571822f41e7a45f3e344f5d667ba7e46b08bfe38190c6996e05cc6271d589340f91b659b04ad279a66aaf29fc5c234107aabf0c722489b48ea64d9be719128f32bc5d56b7b2f19e2bb2e0134714913c7279b194400e5b22f24cbaa2b825742be756fbb257747c6677c2e141f437e9bb64d7ff4c573fde01df4f40d20a5ba4f23853dc33d89cdfc237721a16212ba8ad2d8d30430713ee477b5187b4ccb50edd3683597d5a99258908fe9e74178a8be41e73ab77fe17889dcd7542db1f8ceaa8c5fe918adf111dfdd605f40c19e93f9ad543d872111ec416aa3cb5e60909c01345&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T12:21:22.635640911+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/3wcb4wer/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T041909Z&X-Goog-Expires=86399&X-Goog-Signature=810610a47880236a95ccb7704d4c0dc16f7a0b7eae65667872dce5f537b288a571822f41e7a45f3e344f5d667ba7e46b08bfe38190c6996e05cc6271d589340f91b659b04ad279a66aaf29fc5c234107aabf0c722489b48ea64d9be719128f32bc5d56b7b2f19e2bb2e0134714913c7279b194400e5b22f24cbaa2b825742be756fbb257747c6677c2e141f437e9bb64d7ff4c573fde01df4f40d20a5ba4f23853dc33d89cdfc237721a16212ba8ad2d8d30430713ee477b5187b4ccb50edd3683597d5a99258908fe9e74178a8be41e73ab77fe17889dcd7542db1f8ceaa8c5fe918adf111dfdd605f40c19e93f9ad543d872111ec416aa3cb5e60909c01345&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:36836->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/3wcb4wer/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T041909Z&X-Goog-Expires=86399&X-Goog-Signature=810610a47880236a95ccb7704d4c0dc16f7a0b7eae65667872dce5f537b288a571822f41e7a45f3e344f5d667ba7e46b08bfe38190c6996e05cc6271d589340f91b659b04ad279a66aaf29fc5c234107aabf0c722489b48ea64d9be719128f32bc5d56b7b2f19e2bb2e0134714913c7279b194400e5b22f24cbaa2b825742be756fbb257747c6677c2e141f437e9bb64d7ff4c573fde01df4f40d20a5ba4f23853dc33d89cdfc237721a16212ba8ad2d8d30430713ee477b5187b4ccb50edd3683597d5a99258908fe9e74178a8be41e73ab77fe17889dcd7542db1f8ceaa8c5fe918adf111dfdd605f40c19e93f9ad543d872111ec416aa3cb5e60909c01345&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T12:21:28.126891685+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/3wcb4wer/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T041909Z&X-Goog-Expires=86399&X-Goog-Signature=810610a47880236a95ccb7704d4c0dc16f7a0b7eae65667872dce5f537b288a571822f41e7a45f3e344f5d667ba7e46b08bfe38190c6996e05cc6271d589340f91b659b04ad279a66aaf29fc5c234107aabf0c722489b48ea64d9be719128f32bc5d56b7b2f19e2bb2e0134714913c7279b194400e5b22f24cbaa2b825742be756fbb257747c6677c2e141f437e9bb64d7ff4c573fde01df4f40d20a5ba4f23853dc33d89cdfc237721a16212ba8ad2d8d30430713ee477b5187b4ccb50edd3683597d5a99258908fe9e74178a8be41e73ab77fe17889dcd7542db1f8ceaa8c5fe918adf111dfdd605f40c19e93f9ad543d872111ec416aa3cb5e60909c01345&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:49190->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/3wcb4wer/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T041909Z&X-Goog-Expires=86399&X-Goog-Signature=810610a47880236a95ccb7704d4c0dc16f7a0b7eae65667872dce5f537b288a571822f41e7a45f3e344f5d667ba7e46b08bfe38190c6996e05cc6271d589340f91b659b04ad279a66aaf29fc5c234107aabf0c722489b48ea64d9be719128f32bc5d56b7b2f19e2bb2e0134714913c7279b194400e5b22f24cbaa2b825742be756fbb257747c6677c2e141f437e9bb64d7ff4c573fde01df4f40d20a5ba4f23853dc33d89cdfc237721a16212ba8ad2d8d30430713ee477b5187b4ccb50edd3683597d5a99258908fe9e74178a8be41e73ab77fe17889dcd7542db1f8ceaa8c5fe918adf111dfdd605f40c19e93f9ad543d872111ec416aa3cb5e60909c01345&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T12:23:35.200988988+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/3wcb4wer/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T041909Z&X-Goog-Expires=86399&X-Goog-Signature=2235d1f23ffa94e46560d8d9f1d99a989e9a6742f86a4808f9aeb398930bff09a6648b165f02c86e3336b2c45dea184c1c00bfc510a911f21d1d3c93af8315df007c4be4d48a40d509eddf912bad9dad6111e122f0c52e4718f52e9920b371ccf7d661507efed6a4dca98d6905e4aa52888a2f967d3b5cb0a7edb86ae58b67aa789d6b6d0baeadacdb94ddddd35daa1abcc27411f73ce268720d2a07f37be8e791e57cb72538e70626bd1fa9f490e39a22eeeb4e0ba62fa31b4209ee026d51d1e750551a6dcbfd81ee69fd25b7a1377e4688b46f170feafeb3a758aa1d9e1b881ee4009e8798452e7af98f262897f880de73f172a6426c917978ef0925e2e600&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:59810->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/3wcb4wer/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T041909Z&X-Goog-Expires=86399&X-Goog-Signature=2235d1f23ffa94e46560d8d9f1d99a989e9a6742f86a4808f9aeb398930bff09a6648b165f02c86e3336b2c45dea184c1c00bfc510a911f21d1d3c93af8315df007c4be4d48a40d509eddf912bad9dad6111e122f0c52e4718f52e9920b371ccf7d661507efed6a4dca98d6905e4aa52888a2f967d3b5cb0a7edb86ae58b67aa789d6b6d0baeadacdb94ddddd35daa1abcc27411f73ce268720d2a07f37be8e791e57cb72538e70626bd1fa9f490e39a22eeeb4e0ba62fa31b4209ee026d51d1e750551a6dcbfd81ee69fd25b7a1377e4688b46f170feafeb3a758aa1d9e1b881ee4009e8798452e7af98f262897f880de73f172a6426c917978ef0925e2e600&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T12:32:39.216496055+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": read tcp 10.1.1.90:51308->104.21.20.172:443: read: connection reset by peer"}
+{"time":"2025-08-13T12:50:59.026820434+08:00","level":"INFO","msg":"stream: closing","id":"3wcb4wer"}
+{"time":"2025-08-13T12:51:02.081328021+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/3wcb4wer/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T045059Z&X-Goog-Expires=86399&X-Goog-Signature=8d0f986d028f03059f3bff491d651565b582576cfdafdfdab6e66ea093da1343802941babac635335ed038e1bbd1238c743082aa2ed1185361180b4f0975e9e8561cd1ac3d40566f176e68fe8dda7d344fdef70b0e12d916df0b76e86443f651201dead7fa09deea52c190864f6ecce2d3e92a995b757b5e0ad59abf85d627116927c6c77b3f1cd267c1e1067f3d75c679404c31711964312ef6c8d5b2b03f7feae1450de7e3ac7842fbde41e5c5cffd757e2e5d8e8ecd410bf9b6256bd6d39d26c3045758a8d88ebb2250e9a4cfe3b7acdcf7c3b56cdf9bf7ac63e04aa186a332dc0859f1c10ee38d9bf18eaa589d01c332f04a428612ef98ade0a0dba09257&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:44048->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/3wcb4wer/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T045059Z&X-Goog-Expires=86399&X-Goog-Signature=8d0f986d028f03059f3bff491d651565b582576cfdafdfdab6e66ea093da1343802941babac635335ed038e1bbd1238c743082aa2ed1185361180b4f0975e9e8561cd1ac3d40566f176e68fe8dda7d344fdef70b0e12d916df0b76e86443f651201dead7fa09deea52c190864f6ecce2d3e92a995b757b5e0ad59abf85d627116927c6c77b3f1cd267c1e1067f3d75c679404c31711964312ef6c8d5b2b03f7feae1450de7e3ac7842fbde41e5c5cffd757e2e5d8e8ecd410bf9b6256bd6d39d26c3045758a8d88ebb2250e9a4cfe3b7acdcf7c3b56cdf9bf7ac63e04aa186a332dc0859f1c10ee38d9bf18eaa589d01c332f04a428612ef98ade0a0dba09257&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T12:51:05.123245787+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-08-13T12:51:09.665755774+08:00","level":"INFO","msg":"handler: closed","stream_id":"3wcb4wer"}
+{"time":"2025-08-13T12:51:09.669627695+08:00","level":"INFO","msg":"sender: closed","stream_id":"3wcb4wer"}
+{"time":"2025-08-13T12:51:09.669650417+08:00","level":"INFO","msg":"stream: closed","id":"3wcb4wer"}

BioReason_new/wandb/run-20250813_121906-3wcb4wer/logs/debug.log

@@ -0,0 +1,23 @@
+2025-08-13 12:19:06,598 INFO    MainThread:2035 [wandb_setup.py:_flush():80] Current SDK version is 0.21.1
+2025-08-13 12:19:06,598 INFO    MainThread:2035 [wandb_setup.py:_flush():80] Configure stats pid to 2035
+2025-08-13 12:19:06,598 INFO    MainThread:2035 [wandb_setup.py:_flush():80] Loading settings from /root/.config/wandb/settings
+2025-08-13 12:19:06,598 INFO    MainThread:2035 [wandb_setup.py:_flush():80] Loading settings from /nas/shared/kilab/wangyujia/BioReason_new/wandb/settings
+2025-08-13 12:19:06,598 INFO    MainThread:2035 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-08-13 12:19:06,598 INFO    MainThread:2035 [wandb_init.py:setup_run_log_directory():703] Logging user logs to /nas/shared/kilab/wangyujia/BioReason_new/wandb/run-20250813_121906-3wcb4wer/logs/debug.log
+2025-08-13 12:19:06,598 INFO    MainThread:2035 [wandb_init.py:setup_run_log_directory():704] Logging internal logs to /nas/shared/kilab/wangyujia/BioReason_new/wandb/run-20250813_121906-3wcb4wer/logs/debug-internal.log
+2025-08-13 12:19:06,599 INFO    MainThread:2035 [wandb_init.py:init():830] calling init triggers
+2025-08-13 12:19:06,599 INFO    MainThread:2035 [wandb_init.py:init():835] wandb.init called with sweep_config: {}
+config: {'text_model_name': '/oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged', 'protein_model_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m', 'qformer_model_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft', 'cache_dir': '/model-weights', 'num_query_tokens': 8, 'train_dataset': '/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl', 'valid_dataset': '/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl', 'eval_dataset': True, 'output_dir': './contrastive_outputs', 'num_epochs': 10, 'batch_size': 32, 'learning_rate': 0.0001, 'weight_decay': 0.01, 'warmup_steps': 1000, 'gradient_accumulation_steps': 1, 'temperature': 0.07, 'freeze_protein_model': True, 'freeze_text_model': True, 'protein_weight': 1.0, 'text_weight': 1.0, 'enable_ptm': True, 'ptm_weight': 1.0, 'max_length_protein': 1024, 'max_length_text': 512, 'num_workers': 8, 'logging_steps': 100, 'eval_steps': 500, 'save_steps': 1000, 'save_total_limit': 3, 'fp16': False, 'bf16': False, 'seed': 42, 'use_wandb': True, 'wandb_project': 'protein-llm-contrastive', 'wandb_entity': None, '_wandb': {}}
+2025-08-13 12:19:06,599 INFO    MainThread:2035 [wandb_init.py:init():871] starting backend
+2025-08-13 12:19:06,809 INFO    MainThread:2035 [wandb_init.py:init():874] sending inform_init request
+2025-08-13 12:19:06,816 INFO    MainThread:2035 [wandb_init.py:init():882] backend started and connected
+2025-08-13 12:19:06,819 INFO    MainThread:2035 [wandb_init.py:init():953] updated telemetry
+2025-08-13 12:19:06,844 INFO    MainThread:2035 [wandb_init.py:init():977] communicating run to backend with 90.0 second timeout
+2025-08-13 12:19:08,807 INFO    MainThread:2035 [wandb_init.py:init():1029] starting run threads in backend
+2025-08-13 12:19:08,949 INFO    MainThread:2035 [wandb_run.py:_console_start():2494] atexit reg
+2025-08-13 12:19:08,949 INFO    MainThread:2035 [wandb_run.py:_redirect():2342] redirect: wrap_raw
+2025-08-13 12:19:08,949 INFO    MainThread:2035 [wandb_run.py:_redirect():2411] Wrapping output streams.
+2025-08-13 12:19:08,949 INFO    MainThread:2035 [wandb_run.py:_redirect():2434] Redirects installed.
+2025-08-13 12:19:08,952 INFO    MainThread:2035 [wandb_init.py:init():1075] run started, returning control to user process
+2025-08-13 12:50:52,264 INFO    MainThread:2035 [wandb_run.py:_config_callback():1380] config_cb None None {'output_dir': './contrastive_outputs', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': True, 'do_predict': False, 'eval_strategy': 'steps', 'prediction_loss_only': False, 'per_device_train_batch_size': 32, 'per_device_eval_batch_size': 32, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'eval_delay': 0, 'torch_empty_cache_steps': None, 'learning_rate': 0.0001, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 10, 'max_steps': -1, 'lr_scheduler_type': 'linear', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.0, 'warmup_steps': 1000, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': './contrastive_outputs/runs/Aug13_12-50-37_dsw-265304-585cc9d768-ckfd7', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 100, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 1000, 'save_total_limit': 3, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'restore_callback_states_from_checkpoint': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 42, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': False, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 500, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': None, 'disable_tqdm': False, 'remove_unused_columns': False, 'label_names': None, 'load_best_model_at_end': True, 'metric_for_best_model': 'eval_avg_recall_at_1', 'greater_is_better': True, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'deepspeed': None, 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['wandb'], 'ddp_find_unused_parameters': False, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_always_push': False, 'hub_revision': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'include_for_metrics': [], 'eval_do_concat_batches': True, 'fp16_backend': 'auto', 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None, 'batch_eval_metrics': False, 'eval_on_start': False, 'use_liger_kernel': False, 'liger_kernel_config': None, 'eval_use_gather_object': False, 'average_tokens_across_devices': False, 'temperature': 0.07, 'freeze_protein_model': True, 'freeze_text_model': True, 'protein_weight': 1.0, 'text_weight': 1.0, 'max_length_protein': 1024, 'max_length_text': 512, 'enable_ptm': True, 'ptm_weight': 1.0}
+2025-08-13 12:50:59,025 INFO    MsgRouterThr:2035 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles.

BioReason_new/wandb/run-20250813_125236-re7fb96i/files/config.yaml

@@ -0,0 +1,458 @@
+_wandb:
+    value:
+        cli_version: 0.21.1
+        e:
+            gef9d42v3ap5mwuxow19xbx613zovmhi:
+                args:
+                    - --text_model_name
+                    - /oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged
+                    - --protein_model_name
+                    - /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m
+                    - --qformer_model_name
+                    - /nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft
+                    - --num_query_tokens
+                    - "8"
+                    - --train_dataset
+                    - /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl
+                    - --valid_dataset
+                    - /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl
+                    - --output_dir
+                    - ./contrastive_outputs
+                    - --num_epochs
+                    - "10"
+                    - --batch_size
+                    - "8"
+                    - --learning_rate
+                    - "1e-4"
+                    - --temperature
+                    - "0.07"
+                    - --freeze_protein_model
+                    - --freeze_text_model
+                    - --enable_ptm
+                    - --max_length_protein
+                    - "1024"
+                    - --max_length_text
+                    - "512"
+                    - --num_workers
+                    - "8"
+                    - --eval_dataset
+                    - --use_wandb
+                    - --wandb_project
+                    - protein-llm-contrastive
+                    - --logging_steps
+                    - "100"
+                    - --eval_steps
+                    - "500"
+                    - --save_steps
+                    - "1000"
+                codePath: wangyujia/BioReason_new/train_contrastive.py
+                codePathLocal: train_contrastive.py
+                cpu_count: 64
+                cpu_count_logical: 64
+                cudaVersion: "12.1"
+                disk:
+                    /:
+                        total: "1623302262784"
+                        used: "1015009280"
+                email: gia0603yucca@gmail.com
+                executable: /root/miniconda3/envs/bioreason/bin/python
+                git:
+                    commit: b8caf406aa1699c788f0ca6e44a1769452c317db
+                    remote: https://github.com/PorUna-byte/PAR.git
+                gpu: NVIDIA A800-SXM4-80GB
+                gpu_count: 8
+                gpu_nvidia:
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-62fcc532-880f-7324-eb13-3b739a38c29b
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-af1197aa-3e7d-a53e-185d-04b1d64164da
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-3e6e551f-2d7d-a27b-654a-da56933d2283
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-a947b64a-c545-ee5e-30e6-b4b0e2dd8fdc
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-ebd04f3f-c437-1b8b-26f2-4b1ebfee93c4
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-53994891-72a6-25be-a4c8-e32c695155b0
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-a927d00a-15c5-31f5-9ae1-9d2e773821a1
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-93d555c4-29a0-cd78-abac-d98cd160d4e1
+                host: dsw-265304-585cc9d768-ckfd7
+                memory:
+                    total: "549755813888"
+                os: Linux-5.10.134-008.18.kangaroo.al8.x86_64-x86_64-with-glibc2.35
+                program: /nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py
+                python: CPython 3.11.0
+                root: /nas/shared/kilab/wangyujia/BioReason_new
+                startedAt: "2025-08-13T04:52:36.303807Z"
+                writerId: gef9d42v3ap5mwuxow19xbx613zovmhi
+        m:
+            - "1": train/global_step
+              "6":
+                - 3
+              "7": []
+            - "2": '*'
+              "5": 1
+              "6":
+                - 1
+              "7": []
+        python_version: 3.11.0
+        t:
+            "1":
+                - 1
+                - 9
+                - 11
+                - 41
+                - 49
+                - 51
+                - 71
+                - 84
+                - 98
+                - 103
+            "2":
+                - 1
+                - 9
+                - 11
+                - 41
+                - 49
+                - 51
+                - 71
+                - 84
+                - 98
+                - 103
+            "3":
+                - 7
+                - 13
+                - 16
+                - 66
+            "4": 3.11.0
+            "5": 0.21.1
+            "6": 4.55.0
+            "9":
+                "1": transformers_trainer
+            "12": 0.21.1
+            "13": linux-x86_64
+accelerator_config:
+    value:
+        dispatch_batches: null
+        even_batches: true
+        gradient_accumulation_kwargs: null
+        non_blocking: false
+        split_batches: false
+        use_seedable_sampler: true
+adafactor:
+    value: false
+adam_beta1:
+    value: 0.9
+adam_beta2:
+    value: 0.999
+adam_epsilon:
+    value: 1e-08
+auto_find_batch_size:
+    value: false
+average_tokens_across_devices:
+    value: false
+batch_eval_metrics:
+    value: false
+batch_size:
+    value: 8
+bf16:
+    value: false
+bf16_full_eval:
+    value: false
+cache_dir:
+    value: /model-weights
+data_seed:
+    value: null
+dataloader_drop_last:
+    value: false
+dataloader_num_workers:
+    value: 8
+dataloader_persistent_workers:
+    value: false
+dataloader_pin_memory:
+    value: true
+dataloader_prefetch_factor:
+    value: null
+ddp_backend:
+    value: null
+ddp_broadcast_buffers:
+    value: null
+ddp_bucket_cap_mb:
+    value: null
+ddp_find_unused_parameters:
+    value: false
+ddp_timeout:
+    value: 1800
+debug:
+    value: []
+deepspeed:
+    value: null
+disable_tqdm:
+    value: false
+do_eval:
+    value: true
+do_predict:
+    value: false
+do_train:
+    value: false
+enable_ptm:
+    value: true
+eval_accumulation_steps:
+    value: null
+eval_dataset:
+    value: true
+eval_delay:
+    value: 0
+eval_do_concat_batches:
+    value: true
+eval_on_start:
+    value: false
+eval_steps:
+    value: 500
+eval_strategy:
+    value: steps
+eval_use_gather_object:
+    value: false
+fp16:
+    value: false
+fp16_backend:
+    value: auto
+fp16_full_eval:
+    value: false
+fp16_opt_level:
+    value: O1
+freeze_protein_model:
+    value: true
+freeze_text_model:
+    value: true
+fsdp:
+    value: []
+fsdp_config:
+    value:
+        min_num_params: 0
+        xla: false
+        xla_fsdp_grad_ckpt: false
+        xla_fsdp_v2: false
+fsdp_min_num_params:
+    value: 0
+fsdp_transformer_layer_cls_to_wrap:
+    value: null
+full_determinism:
+    value: false
+gradient_accumulation_steps:
+    value: 1
+gradient_checkpointing:
+    value: false
+gradient_checkpointing_kwargs:
+    value: null
+greater_is_better:
+    value: true
+group_by_length:
+    value: false
+half_precision_backend:
+    value: auto
+hub_always_push:
+    value: false
+hub_model_id:
+    value: null
+hub_private_repo:
+    value: null
+hub_revision:
+    value: null
+hub_strategy:
+    value: every_save
+hub_token:
+    value: <HUB_TOKEN>
+ignore_data_skip:
+    value: false
+include_for_metrics:
+    value: []
+include_inputs_for_metrics:
+    value: false
+include_num_input_tokens_seen:
+    value: false
+include_tokens_per_second:
+    value: false
+jit_mode_eval:
+    value: false
+label_names:
+    value: null
+label_smoothing_factor:
+    value: 0
+learning_rate:
+    value: 0.0001
+length_column_name:
+    value: length
+liger_kernel_config:
+    value: null
+load_best_model_at_end:
+    value: true
+local_rank:
+    value: 0
+log_level:
+    value: passive
+log_level_replica:
+    value: warning
+log_on_each_node:
+    value: true
+logging_dir:
+    value: ./contrastive_outputs/runs/Aug13_12-52-45_dsw-265304-585cc9d768-ckfd7
+logging_first_step:
+    value: false
+logging_nan_inf_filter:
+    value: true
+logging_steps:
+    value: 100
+logging_strategy:
+    value: steps
+lr_scheduler_type:
+    value: linear
+max_grad_norm:
+    value: 1
+max_length_protein:
+    value: 1024
+max_length_text:
+    value: 512
+max_steps:
+    value: -1
+metric_for_best_model:
+    value: eval_avg_recall_at_1
+mp_parameters:
+    value: ""
+neftune_noise_alpha:
+    value: null
+no_cuda:
+    value: false
+num_epochs:
+    value: 10
+num_query_tokens:
+    value: 8
+num_train_epochs:
+    value: 10
+num_workers:
+    value: 8
+optim:
+    value: adamw_torch
+optim_args:
+    value: null
+optim_target_modules:
+    value: null
+output_dir:
+    value: ./contrastive_outputs
+overwrite_output_dir:
+    value: false
+past_index:
+    value: -1
+per_device_eval_batch_size:
+    value: 8
+per_device_train_batch_size:
+    value: 8
+per_gpu_eval_batch_size:
+    value: null
+per_gpu_train_batch_size:
+    value: null
+prediction_loss_only:
+    value: false
+protein_model_name:
+    value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m
+protein_weight:
+    value: 1
+ptm_weight:
+    value: 1
+push_to_hub:
+    value: false
+push_to_hub_model_id:
+    value: null
+push_to_hub_organization:
+    value: null
+push_to_hub_token:
+    value: <PUSH_TO_HUB_TOKEN>
+qformer_model_name:
+    value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft
+ray_scope:
+    value: last
+remove_unused_columns:
+    value: false
+report_to:
+    value:
+        - wandb
+restore_callback_states_from_checkpoint:
+    value: false
+resume_from_checkpoint:
+    value: null
+run_name:
+    value: null
+save_on_each_node:
+    value: false
+save_only_model:
+    value: false
+save_safetensors:
+    value: true
+save_steps:
+    value: 1000
+save_strategy:
+    value: steps
+save_total_limit:
+    value: 3
+seed:
+    value: 42
+skip_memory_metrics:
+    value: true
+temperature:
+    value: 0.07
+text_model_name:
+    value: /oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged
+text_weight:
+    value: 1
+tf32:
+    value: null
+torch_compile:
+    value: false
+torch_compile_backend:
+    value: null
+torch_compile_mode:
+    value: null
+torch_empty_cache_steps:
+    value: null
+torchdynamo:
+    value: null
+tpu_metrics_debug:
+    value: false
+tpu_num_cores:
+    value: null
+train_dataset:
+    value: /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl
+use_cpu:
+    value: false
+use_ipex:
+    value: false
+use_legacy_prediction_loop:
+    value: false
+use_liger_kernel:
+    value: false
+use_mps_device:
+    value: false
+use_wandb:
+    value: true
+valid_dataset:
+    value: /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl
+wandb_entity:
+    value: null
+wandb_project:
+    value: protein-llm-contrastive
+warmup_ratio:
+    value: 0
+warmup_steps:
+    value: 1000
+weight_decay:
+    value: 0.01

BioReason_new/wandb/run-20250813_125236-re7fb96i/files/output.log

@@ -0,0 +1,57 @@
+Loading model...
+Loading checkpoint shards: 100%|██████████████████████████████████████████████████████| 4/4 [00:01<00:00,  2.43it/s]
+Some weights of EsmModel were not initialized from the model checkpoint at /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m and are newly initialized: ['pooler.dense.bias', 'pooler.dense.weight']
+You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
+Loading datasets...
+Training dataset size: 430595
+Eval dataset size: 10000
+Initializing enhanced trainer...
+[2025-08-13 12:52:50,263] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/_distutils_hack/__init__.py:53: UserWarning: Reliance on distutils from stdlib is deprecated. Users must rely on setuptools to provide the distutils module. Avoid importing distutils or import setuptools first, and avoid setting SETUPTOOLS_USE_DISTUTILS=stdlib. Register concerns at https://github.com/pypa/setuptools/issues/new?template=distutils-deprecation.yml
+  warnings.warn(
+[2025-08-13 12:52:51,304] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False
+Starting enhanced contrastive training...
+- Contrastive learning enabled
+- Protein-text matching: enabled
+- Temperature: 0.07
+- PTM weight: 1.0
+  0%|                                                                                     | 0/67290 [00:00<?, ?it/s]Traceback (most recent call last):
+  File "/nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py", line 552, in <module>
+    trainer = main(args)
+              ^^^^^^^^^^
+  File "/nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py", line 399, in main
+    trainer.train()
+  File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/trainer.py", line 2238, in train
+    return inner_training_loop(
+           ^^^^^^^^^^^^^^^^^^^^
+  File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/trainer.py", line 2582, in _inner_training_loop
+    tr_loss_step = self.training_step(model, inputs, num_items_in_batch)
+                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/trainer.py", line 3796, in training_step
+    loss = self.compute_loss(model, inputs, num_items_in_batch=num_items_in_batch)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/nas/shared/kilab/wangyujia/BioReason_new/bioreason/trainer/contrast_trainer_new.py", line 289, in compute_loss
+    protein_features = model.get_protein_features(protein_sequences)  # [B, num_queries, embed_dim]
+                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/nas/shared/kilab/wangyujia/BioReason_new/bioreason/models/protein_llm.py", line 429, in get_protein_features
+    protein_features = self.protein_projection(
+                       ^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1762, in _call_impl
+    return forward_call(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/nas/shared/kilab/wangyujia/BioReason_new/bioreason/models/protein_llm.py", line 98, in forward
+    outputs = self.qformer(
+              ^^^^^^^^^^^^^
+  File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1762, in _call_impl
+    return forward_call(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/models/bert/modeling_bert.py", line 937, in forward
+    buffered_token_type_ids_expanded = buffered_token_type_ids.expand(batch_size, seq_length)
+                                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+RuntimeError: The expanded size of the tensor (967) must match the existing size (512) at non-singleton dimension 1.  Target sizes: [64, 967].  Tensor sizes: [1, 512]

BioReason_new/wandb/run-20250813_125236-re7fb96i/files/requirements.txt

@@ -0,0 +1,233 @@
+jupyterlab_pygments==0.3.0
+depyf==0.19.0
+av==15.0.0
+fastapi==0.116.1
+ray==2.48.0
+cachetools==6.1.0
+prometheus_client==0.22.1
+requests==2.32.4
+jupyter-lsp==2.2.6
+jupyter-events==0.12.0
+anyio==4.10.0
+pybase64==1.4.2
+fastjsonschema==2.21.1
+cloudpickle==3.1.1
+sentry-sdk==2.34.1
+torchmetrics==1.8.1
+wheel==0.45.1
+idna==3.10
+defusedxml==0.7.1
+starlette==0.47.2
+python-dotenv==1.1.1
+comm==0.2.3
+multidict==6.6.3
+python-json-logger==3.3.0
+jupyter_server==2.16.0
+nvidia-nvtx-cu12==12.6.77
+websockets==15.0.1
+referencing==0.36.2
+gguf==0.17.1
+lm-format-enforcer==0.10.12
+xformers==0.0.31
+ninja==1.11.1.4
+beautifulsoup4==4.13.4
+dnspython==2.7.0
+certifi==2025.8.3
+markdown-it-py==3.0.0
+stack-data==0.6.3
+nvidia-cusparselt-cu12==0.6.3
+networkx==3.5
+aiohappyeyeballs==2.6.1
+nvidia-cuda-cupti-cu12==12.6.80
+llguidance==0.7.30
+rich==14.1.0
+outlines_core==0.2.10
+nvidia-nccl-cu12==2.26.2
+pandas==2.3.1
+pydantic_core==2.33.2
+argon2-cffi==25.1.0
+babel==2.17.0
+isoduration==20.11.0
+uvloop==0.21.0
+json5==0.12.0
+widgetsnbextension==4.0.14
+aiohttp==3.12.15
+notebook_shim==0.2.4
+pandocfilters==1.5.1
+vllm==0.10.0
+blake3==1.0.5
+diskcache==5.6.3
+h11==0.16.0
+attrs==25.3.0
+nvidia-cusolver-cu12==11.7.1.2
+click==8.2.1
+cupy-cuda12x==13.5.1
+jedi==0.19.2
+py-cpuinfo==9.0.0
+overrides==7.7.0
+pytz==2025.2
+protobuf==6.31.1
+dill==0.3.8
+ipykernel==6.30.1
+frozenlist==1.7.0
+huggingface-hub==0.34.4
+rignore==0.6.4
+mistune==3.1.3
+sniffio==1.3.1
+httpx==0.28.1
+sentencepiece==0.2.0
+typing_extensions==4.14.1
+typing-inspection==0.4.1
+async-lru==2.0.5
+bioreason==0.1.0
+charset-normalizer==3.4.3
+llvmlite==0.44.0
+safetensors==0.6.2
+fqdn==1.5.1
+peft==0.17.0
+fastapi-cli==0.0.8
+torch==2.7.1
+jsonschema==4.25.0
+torchvision==0.22.1
+parso==0.8.4
+pillow==11.3.0
+pycountry==24.6.1
+scipy==1.16.1
+wandb==0.21.1
+pip==25.1
+wcwidth==0.2.13
+mistral_common==1.8.3
+nvidia-cudnn-cu12==9.5.1.17
+asttokens==3.0.0
+numpy==2.2.6
+uvicorn==0.35.0
+jupyter_client==8.6.3
+debugpy==1.8.16
+fastapi-cloud-cli==0.1.5
+compressed-tensors==0.10.2
+GitPython==3.1.45
+nvidia-ml-py==13.580.65
+tiktoken==0.11.0
+partial-json-parser==0.2.1.1.post6
+prompt_toolkit==3.0.51
+uri-template==1.3.0
+pyarrow==21.0.0
+nvidia-cuda-nvrtc-cu12==12.6.77
+propcache==0.3.2
+matplotlib-inline==0.1.7
+pure_eval==0.2.3
+nest-asyncio==1.6.0
+deepspeed==0.17.4
+einops==0.8.1
+triton==3.3.1
+trl==0.21.0
+pyzmq==27.0.1
+httptools==0.6.4
+ipython==9.4.0
+packaging==25.0
+hjson==3.1.0
+soxr==0.5.0.post1
+webcolors==24.11.1
+PyYAML==6.0.2
+ipywidgets==8.1.7
+lark==1.2.2
+python-multipart==0.0.20
+tinycss2==1.4.0
+mpmath==1.3.0
+psutil==7.0.0
+nbformat==5.10.4
+qwen-vl-utils==0.0.11
+notebook==7.4.5
+jupyterlab==4.4.5
+transformers==4.55.0
+urllib3==2.5.0
+terminado==0.18.1
+xgrammar==0.1.21
+xxhash==3.5.0
+tzdata==2025.2
+jupyterlab_widgets==3.0.15
+aiosignal==1.4.0
+annotated-types==0.7.0
+nvidia-nvjitlink-cu12==12.6.85
+httpcore==1.0.9
+pexpect==4.9.0
+tornado==6.5.2
+regex==2025.7.33
+filelock==3.18.0
+nvidia-cuda-runtime-cu12==12.6.77
+openai==1.90.0
+Pygments==2.19.2
+cffi==1.17.1
+nvidia-cufft-cu12==11.3.0.4
+rpds-py==0.27.0
+rich-toolkit==0.15.0
+bleach==6.2.0
+sympy==1.14.0
+python-dateutil==2.9.0.post0
+watchfiles==1.1.0
+nbclient==0.10.2
+hf-xet==1.1.7
+Send2Trash==1.8.3
+smmap==5.0.2
+tokenizers==0.21.4
+yarl==1.20.1
+MarkupSafe==3.0.2
+prometheus-fastapi-instrumentator==7.1.0
+rfc3986-validator==0.1.1
+bitsandbytes==0.46.1
+nvidia-cufile-cu12==1.11.1.6
+fsspec==2025.3.0
+executing==2.2.0
+pydantic==2.11.7
+jupyter_core==5.8.1
+msgspec==0.19.0
+nvidia-cusparse-cu12==12.5.4.2
+opencv-python-headless==4.12.0.88
+webencodings==0.5.1
+ptyprocess==0.7.0
+pytorch-lightning==2.5.2
+Jinja2==3.1.6
+websocket-client==1.8.0
+jsonpointer==3.0.0
+traitlets==5.14.3
+numba==0.61.2
+soupsieve==2.7
+cbor2==5.6.5
+multiprocess==0.70.16
+email_validator==2.2.0
+distro==1.9.0
+accelerate==1.10.0
+torchaudio==2.7.1
+jiter==0.10.0
+rfc3339-validator==0.1.4
+jupyter==1.1.1
+typer==0.16.0
+gitdb==4.0.12
+types-python-dateutil==2.9.0.20250809
+soundfile==0.13.1
+astor==0.8.1
+shellingham==1.5.4
+jupyterlab_server==2.27.3
+nvidia-curand-cu12==10.3.7.77
+jupyter-console==6.6.3
+msgpack==1.1.1
+six==1.17.0
+pycparser==2.22
+arrow==1.3.0
+tqdm==4.67.1
+lightning-utilities==0.15.2
+interegular==0.3.3
+rfc3987-syntax==1.1.0
+ipython_pygments_lexers==1.1.1
+datasets==4.0.0
+jupyter_server_terminals==0.5.3
+platformdirs==4.3.8
+pydantic-extra-types==2.10.5
+decorator==5.2.1
+mdurl==0.1.2
+nvidia-cublas-cu12==12.6.4.1
+jsonschema-specifications==2025.4.1
+setuptools==78.1.1
+fastrlock==0.8.3
+argon2-cffi-bindings==25.1.0
+nbconvert==7.16.6

BioReason_new/wandb/run-20250813_125236-re7fb96i/files/wandb-metadata.json

@@ -0,0 +1,116 @@
+{
+  "os": "Linux-5.10.134-008.18.kangaroo.al8.x86_64-x86_64-with-glibc2.35",
+  "python": "CPython 3.11.0",
+  "startedAt": "2025-08-13T04:52:36.303807Z",
+  "args": [
+    "--text_model_name",
+    "/oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged",
+    "--protein_model_name",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m",
+    "--qformer_model_name",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft",
+    "--num_query_tokens",
+    "8",
+    "--train_dataset",
+    "/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl",
+    "--valid_dataset",
+    "/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl",
+    "--output_dir",
+    "./contrastive_outputs",
+    "--num_epochs",
+    "10",
+    "--batch_size",
+    "8",
+    "--learning_rate",
+    "1e-4",
+    "--temperature",
+    "0.07",
+    "--freeze_protein_model",
+    "--freeze_text_model",
+    "--enable_ptm",
+    "--max_length_protein",
+    "1024",
+    "--max_length_text",
+    "512",
+    "--num_workers",
+    "8",
+    "--eval_dataset",
+    "--use_wandb",
+    "--wandb_project",
+    "protein-llm-contrastive",
+    "--logging_steps",
+    "100",
+    "--eval_steps",
+    "500",
+    "--save_steps",
+    "1000"
+  ],
+  "program": "/nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py",
+  "codePath": "wangyujia/BioReason_new/train_contrastive.py",
+  "codePathLocal": "train_contrastive.py",
+  "git": {
+    "remote": "https://github.com/PorUna-byte/PAR.git",
+    "commit": "b8caf406aa1699c788f0ca6e44a1769452c317db"
+  },
+  "email": "gia0603yucca@gmail.com",
+  "root": "/nas/shared/kilab/wangyujia/BioReason_new",
+  "host": "dsw-265304-585cc9d768-ckfd7",
+  "executable": "/root/miniconda3/envs/bioreason/bin/python",
+  "cpu_count": 64,
+  "cpu_count_logical": 64,
+  "gpu": "NVIDIA A800-SXM4-80GB",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1623302262784",
+      "used": "1015009280"
+    }
+  },
+  "memory": {
+    "total": "549755813888"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-62fcc532-880f-7324-eb13-3b739a38c29b"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-af1197aa-3e7d-a53e-185d-04b1d64164da"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-3e6e551f-2d7d-a27b-654a-da56933d2283"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-a947b64a-c545-ee5e-30e6-b4b0e2dd8fdc"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-ebd04f3f-c437-1b8b-26f2-4b1ebfee93c4"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-53994891-72a6-25be-a4c8-e32c695155b0"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-a927d00a-15c5-31f5-9ae1-9d2e773821a1"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-93d555c4-29a0-cd78-abac-d98cd160d4e1"
+    }
+  ],
+  "cudaVersion": "12.1",
+  "writerId": "gef9d42v3ap5mwuxow19xbx613zovmhi"
+}

BioReason_new/wandb/run-20250813_125236-re7fb96i/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":14},"_runtime":14}

BioReason_new/wandb/run-20250813_125236-re7fb96i/logs/debug-internal.log

@@ -0,0 +1,18 @@
+{"time":"2025-08-13T12:52:36.589616991+08:00","level":"INFO","msg":"stream: starting","core version":"0.21.1"}
+{"time":"2025-08-13T12:52:38.561487318+08:00","level":"INFO","msg":"stream: created new stream","id":"re7fb96i"}
+{"time":"2025-08-13T12:52:38.561535116+08:00","level":"INFO","msg":"stream: started","id":"re7fb96i"}
+{"time":"2025-08-13T12:52:38.56159066+08:00","level":"INFO","msg":"writer: started","stream_id":"re7fb96i"}
+{"time":"2025-08-13T12:52:38.561602542+08:00","level":"INFO","msg":"sender: started","stream_id":"re7fb96i"}
+{"time":"2025-08-13T12:52:38.561649125+08:00","level":"INFO","msg":"handler: started","stream_id":"re7fb96i"}
+{"time":"2025-08-13T12:52:44.06545967+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/re7fb96i/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T045241Z&X-Goog-Expires=86399&X-Goog-Signature=5ab95aad544e88ddad2af3439c4ef5fd6cad599df4041ec7a3094e1270d666c27d4c5d84bd7bae3293680630f98ef7873f55e59c7cbb9c983f77007539bc165573b1876eafbf04a7703c05db86dbf5d62862b67515b993edec4fb89877428a0f6ff39beefc2c2d965c832b1f9bbd8cd7ee52831df86c571243838706679984faa939e743fa1c888f4a60beccecf15caba84a74ff0dd96023d8f89371f4b0849496de2439804149a39b3f1dbb3b1a7a3702a46d55b34fd4dc64fc181ad0eaa42bf08cc9856827be672f72262b6c85e3d44dc6efd662c5f2f1f67840978533a27c4c6ed0e131632f8aa8306e955bc031ef6e95e3d62b8eea848f0e2d36ceccfa19&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:53438->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/re7fb96i/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T045241Z&X-Goog-Expires=86399&X-Goog-Signature=5ab95aad544e88ddad2af3439c4ef5fd6cad599df4041ec7a3094e1270d666c27d4c5d84bd7bae3293680630f98ef7873f55e59c7cbb9c983f77007539bc165573b1876eafbf04a7703c05db86dbf5d62862b67515b993edec4fb89877428a0f6ff39beefc2c2d965c832b1f9bbd8cd7ee52831df86c571243838706679984faa939e743fa1c888f4a60beccecf15caba84a74ff0dd96023d8f89371f4b0849496de2439804149a39b3f1dbb3b1a7a3702a46d55b34fd4dc64fc181ad0eaa42bf08cc9856827be672f72262b6c85e3d44dc6efd662c5f2f1f67840978533a27c4c6ed0e131632f8aa8306e955bc031ef6e95e3d62b8eea848f0e2d36ceccfa19&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T12:52:46.994692315+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/re7fb96i/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T045241Z&X-Goog-Expires=86399&X-Goog-Signature=5ab95aad544e88ddad2af3439c4ef5fd6cad599df4041ec7a3094e1270d666c27d4c5d84bd7bae3293680630f98ef7873f55e59c7cbb9c983f77007539bc165573b1876eafbf04a7703c05db86dbf5d62862b67515b993edec4fb89877428a0f6ff39beefc2c2d965c832b1f9bbd8cd7ee52831df86c571243838706679984faa939e743fa1c888f4a60beccecf15caba84a74ff0dd96023d8f89371f4b0849496de2439804149a39b3f1dbb3b1a7a3702a46d55b34fd4dc64fc181ad0eaa42bf08cc9856827be672f72262b6c85e3d44dc6efd662c5f2f1f67840978533a27c4c6ed0e131632f8aa8306e955bc031ef6e95e3d62b8eea848f0e2d36ceccfa19&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:46682->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/re7fb96i/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T045241Z&X-Goog-Expires=86399&X-Goog-Signature=5ab95aad544e88ddad2af3439c4ef5fd6cad599df4041ec7a3094e1270d666c27d4c5d84bd7bae3293680630f98ef7873f55e59c7cbb9c983f77007539bc165573b1876eafbf04a7703c05db86dbf5d62862b67515b993edec4fb89877428a0f6ff39beefc2c2d965c832b1f9bbd8cd7ee52831df86c571243838706679984faa939e743fa1c888f4a60beccecf15caba84a74ff0dd96023d8f89371f4b0849496de2439804149a39b3f1dbb3b1a7a3702a46d55b34fd4dc64fc181ad0eaa42bf08cc9856827be672f72262b6c85e3d44dc6efd662c5f2f1f67840978533a27c4c6ed0e131632f8aa8306e955bc031ef6e95e3d62b8eea848f0e2d36ceccfa19&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T12:52:51.612089813+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/re7fb96i/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T045241Z&X-Goog-Expires=86399&X-Goog-Signature=5ab95aad544e88ddad2af3439c4ef5fd6cad599df4041ec7a3094e1270d666c27d4c5d84bd7bae3293680630f98ef7873f55e59c7cbb9c983f77007539bc165573b1876eafbf04a7703c05db86dbf5d62862b67515b993edec4fb89877428a0f6ff39beefc2c2d965c832b1f9bbd8cd7ee52831df86c571243838706679984faa939e743fa1c888f4a60beccecf15caba84a74ff0dd96023d8f89371f4b0849496de2439804149a39b3f1dbb3b1a7a3702a46d55b34fd4dc64fc181ad0eaa42bf08cc9856827be672f72262b6c85e3d44dc6efd662c5f2f1f67840978533a27c4c6ed0e131632f8aa8306e955bc031ef6e95e3d62b8eea848f0e2d36ceccfa19&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:57324->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/re7fb96i/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T045241Z&X-Goog-Expires=86399&X-Goog-Signature=5ab95aad544e88ddad2af3439c4ef5fd6cad599df4041ec7a3094e1270d666c27d4c5d84bd7bae3293680630f98ef7873f55e59c7cbb9c983f77007539bc165573b1876eafbf04a7703c05db86dbf5d62862b67515b993edec4fb89877428a0f6ff39beefc2c2d965c832b1f9bbd8cd7ee52831df86c571243838706679984faa939e743fa1c888f4a60beccecf15caba84a74ff0dd96023d8f89371f4b0849496de2439804149a39b3f1dbb3b1a7a3702a46d55b34fd4dc64fc181ad0eaa42bf08cc9856827be672f72262b6c85e3d44dc6efd662c5f2f1f67840978533a27c4c6ed0e131632f8aa8306e955bc031ef6e95e3d62b8eea848f0e2d36ceccfa19&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T12:52:55.303102498+08:00","level":"INFO","msg":"stream: closing","id":"re7fb96i"}
+{"time":"2025-08-13T12:52:57.319008656+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/re7fb96i/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T045256Z&X-Goog-Expires=86399&X-Goog-Signature=6cd70ba74845b1ae0f9a8f83f1a4441043fa503279d42e4f7e55f638f643835d7270b1d0c42f0cb81df5618f3250c30a10fd02484a7951eeaac60df9b1912a487c976b34dc34f276fd38be0bd372a32e1951ea232361be43a8f78feb4c52042f05a5e7fe515d2e96d6e6187378eee4ff85ed4df1639c554abddd23693fc3c951722445f98a3d6e62052d0c35daa557227cd76aa9efbebe6191e1ae0d7b4ed7241abeab321d09c60a591aeed73d864d6b86496848ab61b6517e66eac19ec2ceaebdb5ae43c1010b820eba8b9e2694683b136e639c2186f348671a910a2f1fa0121439bafd138ff38092414668a7fb59e535b1808cefa7fc42747b4b7b3ba26bc5&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:41368->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/re7fb96i/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T045256Z&X-Goog-Expires=86399&X-Goog-Signature=6cd70ba74845b1ae0f9a8f83f1a4441043fa503279d42e4f7e55f638f643835d7270b1d0c42f0cb81df5618f3250c30a10fd02484a7951eeaac60df9b1912a487c976b34dc34f276fd38be0bd372a32e1951ea232361be43a8f78feb4c52042f05a5e7fe515d2e96d6e6187378eee4ff85ed4df1639c554abddd23693fc3c951722445f98a3d6e62052d0c35daa557227cd76aa9efbebe6191e1ae0d7b4ed7241abeab321d09c60a591aeed73d864d6b86496848ab61b6517e66eac19ec2ceaebdb5ae43c1010b820eba8b9e2694683b136e639c2186f348671a910a2f1fa0121439bafd138ff38092414668a7fb59e535b1808cefa7fc42747b4b7b3ba26bc5&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T12:54:51.198748634+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/re7fb96i/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T045241Z&X-Goog-Expires=86399&X-Goog-Signature=66a468dbcc807521a33850cbde9ed7c890df5238e496201fe7c802709c40b88899c9145fa156b9e5ff9d60d4fce2e40a3550b45198637bc47d62374ab504d81be8116a0d981e038945eb827e976a9e196c72459cd5324c1f95a29ca9f03c9448c37efe4c69f9fa20e463fca0383d500a543ff5a0d68c4b29831d61803b4640eb4ead83681cbfd46b82dbbef572a174fc49c715ff3886d78f6490b7d8162b8479cb62884c5951ff89d46f926ecee4548e03b9460e45e0f3dade083311e0770dcaa8252111666869262f4647000463a4c49491d12f9aa50488cf2d22d1809f5b77a00e9ee23657fcc7420c084261ed55d8370ac97e8f523eb950a7827b25ec4736&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:53304->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/re7fb96i/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T045241Z&X-Goog-Expires=86399&X-Goog-Signature=66a468dbcc807521a33850cbde9ed7c890df5238e496201fe7c802709c40b88899c9145fa156b9e5ff9d60d4fce2e40a3550b45198637bc47d62374ab504d81be8116a0d981e038945eb827e976a9e196c72459cd5324c1f95a29ca9f03c9448c37efe4c69f9fa20e463fca0383d500a543ff5a0d68c4b29831d61803b4640eb4ead83681cbfd46b82dbbef572a174fc49c715ff3886d78f6490b7d8162b8479cb62884c5951ff89d46f926ecee4548e03b9460e45e0f3dade083311e0770dcaa8252111666869262f4647000463a4c49491d12f9aa50488cf2d22d1809f5b77a00e9ee23657fcc7420c084261ed55d8370ac97e8f523eb950a7827b25ec4736&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T12:54:53.616083501+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/re7fb96i/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T045241Z&X-Goog-Expires=86399&X-Goog-Signature=66a468dbcc807521a33850cbde9ed7c890df5238e496201fe7c802709c40b88899c9145fa156b9e5ff9d60d4fce2e40a3550b45198637bc47d62374ab504d81be8116a0d981e038945eb827e976a9e196c72459cd5324c1f95a29ca9f03c9448c37efe4c69f9fa20e463fca0383d500a543ff5a0d68c4b29831d61803b4640eb4ead83681cbfd46b82dbbef572a174fc49c715ff3886d78f6490b7d8162b8479cb62884c5951ff89d46f926ecee4548e03b9460e45e0f3dade083311e0770dcaa8252111666869262f4647000463a4c49491d12f9aa50488cf2d22d1809f5b77a00e9ee23657fcc7420c084261ed55d8370ac97e8f523eb950a7827b25ec4736&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:51230->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/re7fb96i/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T045241Z&X-Goog-Expires=86399&X-Goog-Signature=66a468dbcc807521a33850cbde9ed7c890df5238e496201fe7c802709c40b88899c9145fa156b9e5ff9d60d4fce2e40a3550b45198637bc47d62374ab504d81be8116a0d981e038945eb827e976a9e196c72459cd5324c1f95a29ca9f03c9448c37efe4c69f9fa20e463fca0383d500a543ff5a0d68c4b29831d61803b4640eb4ead83681cbfd46b82dbbef572a174fc49c715ff3886d78f6490b7d8162b8479cb62884c5951ff89d46f926ecee4548e03b9460e45e0f3dade083311e0770dcaa8252111666869262f4647000463a4c49491d12f9aa50488cf2d22d1809f5b77a00e9ee23657fcc7420c084261ed55d8370ac97e8f523eb950a7827b25ec4736&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T12:55:07.705724989+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/re7fb96i/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T045256Z&X-Goog-Expires=86399&X-Goog-Signature=21e8bf224738001dd556d09ec67e19b73e3bbf8672f24a2a3d1f83b54e794b40198452cb9f3b3e9e7726111cc5a89448372dc9a2f3c8414180b3ab89bf6dda7eb7ed58bbe92da4fb684762d530af66a5ef72fd8b746c34892dd054fa7fe727660862a6e5f11ace5ce16d47e03034dc5a8f4bd240f3c58691a1f8b093baef3e4029aa382a17fac6d918e2b26fb8a2b395cc3fb091db2e25ae7d683b992ab3828622593250c82968565c439a973999890b21c772f52c76ce6ec08863a755e829a7531cec69482c34a6956b56e7f83ca69dad5e7e270c21b429968530587522425487f8d0b5de513173609a157f46dac948cafdc901306b271f645af87c0f5c0399&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:34228->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/re7fb96i/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T045256Z&X-Goog-Expires=86399&X-Goog-Signature=21e8bf224738001dd556d09ec67e19b73e3bbf8672f24a2a3d1f83b54e794b40198452cb9f3b3e9e7726111cc5a89448372dc9a2f3c8414180b3ab89bf6dda7eb7ed58bbe92da4fb684762d530af66a5ef72fd8b746c34892dd054fa7fe727660862a6e5f11ace5ce16d47e03034dc5a8f4bd240f3c58691a1f8b093baef3e4029aa382a17fac6d918e2b26fb8a2b395cc3fb091db2e25ae7d683b992ab3828622593250c82968565c439a973999890b21c772f52c76ce6ec08863a755e829a7531cec69482c34a6956b56e7f83ca69dad5e7e270c21b429968530587522425487f8d0b5de513173609a157f46dac948cafdc901306b271f645af87c0f5c0399&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T12:55:10.365569392+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-08-13T12:55:12.589325353+08:00","level":"INFO","msg":"handler: closed","stream_id":"re7fb96i"}
+{"time":"2025-08-13T12:55:12.5926135+08:00","level":"INFO","msg":"sender: closed","stream_id":"re7fb96i"}
+{"time":"2025-08-13T12:55:12.592621763+08:00","level":"INFO","msg":"stream: closed","id":"re7fb96i"}

BioReason_new/wandb/run-20250813_125236-re7fb96i/logs/debug.log

@@ -0,0 +1,23 @@
+2025-08-13 12:52:36,370 INFO    MainThread:4711 [wandb_setup.py:_flush():80] Current SDK version is 0.21.1
+2025-08-13 12:52:36,370 INFO    MainThread:4711 [wandb_setup.py:_flush():80] Configure stats pid to 4711
+2025-08-13 12:52:36,370 INFO    MainThread:4711 [wandb_setup.py:_flush():80] Loading settings from /root/.config/wandb/settings
+2025-08-13 12:52:36,370 INFO    MainThread:4711 [wandb_setup.py:_flush():80] Loading settings from /nas/shared/kilab/wangyujia/BioReason_new/wandb/settings
+2025-08-13 12:52:36,370 INFO    MainThread:4711 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-08-13 12:52:36,370 INFO    MainThread:4711 [wandb_init.py:setup_run_log_directory():703] Logging user logs to /nas/shared/kilab/wangyujia/BioReason_new/wandb/run-20250813_125236-re7fb96i/logs/debug.log
+2025-08-13 12:52:36,370 INFO    MainThread:4711 [wandb_init.py:setup_run_log_directory():704] Logging internal logs to /nas/shared/kilab/wangyujia/BioReason_new/wandb/run-20250813_125236-re7fb96i/logs/debug-internal.log
+2025-08-13 12:52:36,370 INFO    MainThread:4711 [wandb_init.py:init():830] calling init triggers
+2025-08-13 12:52:36,370 INFO    MainThread:4711 [wandb_init.py:init():835] wandb.init called with sweep_config: {}
+config: {'text_model_name': '/oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged', 'protein_model_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m', 'qformer_model_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft', 'cache_dir': '/model-weights', 'num_query_tokens': 8, 'train_dataset': '/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl', 'valid_dataset': '/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl', 'eval_dataset': True, 'output_dir': './contrastive_outputs', 'num_epochs': 10, 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 0.01, 'warmup_steps': 1000, 'gradient_accumulation_steps': 1, 'temperature': 0.07, 'freeze_protein_model': True, 'freeze_text_model': True, 'protein_weight': 1.0, 'text_weight': 1.0, 'enable_ptm': True, 'ptm_weight': 1.0, 'max_length_protein': 1024, 'max_length_text': 512, 'num_workers': 8, 'logging_steps': 100, 'eval_steps': 500, 'save_steps': 1000, 'save_total_limit': 3, 'fp16': False, 'bf16': False, 'seed': 42, 'use_wandb': True, 'wandb_project': 'protein-llm-contrastive', 'wandb_entity': None, '_wandb': {}}
+2025-08-13 12:52:36,370 INFO    MainThread:4711 [wandb_init.py:init():871] starting backend
+2025-08-13 12:52:36,579 INFO    MainThread:4711 [wandb_init.py:init():874] sending inform_init request
+2025-08-13 12:52:36,585 INFO    MainThread:4711 [wandb_init.py:init():882] backend started and connected
+2025-08-13 12:52:36,586 INFO    MainThread:4711 [wandb_init.py:init():953] updated telemetry
+2025-08-13 12:52:36,614 INFO    MainThread:4711 [wandb_init.py:init():977] communicating run to backend with 90.0 second timeout
+2025-08-13 12:52:41,160 INFO    MainThread:4711 [wandb_init.py:init():1029] starting run threads in backend
+2025-08-13 12:52:41,294 INFO    MainThread:4711 [wandb_run.py:_console_start():2494] atexit reg
+2025-08-13 12:52:41,294 INFO    MainThread:4711 [wandb_run.py:_redirect():2342] redirect: wrap_raw
+2025-08-13 12:52:41,294 INFO    MainThread:4711 [wandb_run.py:_redirect():2411] Wrapping output streams.
+2025-08-13 12:52:41,294 INFO    MainThread:4711 [wandb_run.py:_redirect():2434] Redirects installed.
+2025-08-13 12:52:41,296 INFO    MainThread:4711 [wandb_init.py:init():1075] run started, returning control to user process
+2025-08-13 12:52:51,565 INFO    MainThread:4711 [wandb_run.py:_config_callback():1380] config_cb None None {'output_dir': './contrastive_outputs', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': True, 'do_predict': False, 'eval_strategy': 'steps', 'prediction_loss_only': False, 'per_device_train_batch_size': 8, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'eval_delay': 0, 'torch_empty_cache_steps': None, 'learning_rate': 0.0001, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 10, 'max_steps': -1, 'lr_scheduler_type': 'linear', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.0, 'warmup_steps': 1000, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': './contrastive_outputs/runs/Aug13_12-52-45_dsw-265304-585cc9d768-ckfd7', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 100, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 1000, 'save_total_limit': 3, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'restore_callback_states_from_checkpoint': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 42, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': False, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 500, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': None, 'disable_tqdm': False, 'remove_unused_columns': False, 'label_names': None, 'load_best_model_at_end': True, 'metric_for_best_model': 'eval_avg_recall_at_1', 'greater_is_better': True, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'deepspeed': None, 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['wandb'], 'ddp_find_unused_parameters': False, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_always_push': False, 'hub_revision': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'include_for_metrics': [], 'eval_do_concat_batches': True, 'fp16_backend': 'auto', 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None, 'batch_eval_metrics': False, 'eval_on_start': False, 'use_liger_kernel': False, 'liger_kernel_config': None, 'eval_use_gather_object': False, 'average_tokens_across_devices': False, 'temperature': 0.07, 'freeze_protein_model': True, 'freeze_text_model': True, 'protein_weight': 1.0, 'text_weight': 1.0, 'max_length_protein': 1024, 'max_length_text': 512, 'enable_ptm': True, 'ptm_weight': 1.0}
+2025-08-13 12:52:55,302 INFO    MsgRouterThr:4711 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles.

BioReason_new/wandb/run-20250813_141819-6d95o7hw/files/config.yaml

@@ -0,0 +1,458 @@
+_wandb:
+    value:
+        cli_version: 0.21.1
+        e:
+            783suewtq8mq4vax242rrsp73rsqcggf:
+                args:
+                    - --text_model_name
+                    - /oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged
+                    - --protein_model_name
+                    - /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m
+                    - --qformer_model_name
+                    - /nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft
+                    - --num_query_tokens
+                    - "8"
+                    - --train_dataset
+                    - /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl
+                    - --valid_dataset
+                    - /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl
+                    - --output_dir
+                    - ./contrastive_outputs
+                    - --num_epochs
+                    - "10"
+                    - --batch_size
+                    - "8"
+                    - --learning_rate
+                    - "1e-4"
+                    - --temperature
+                    - "0.07"
+                    - --freeze_protein_model
+                    - --freeze_text_model
+                    - --enable_ptm
+                    - --max_length_protein
+                    - "1024"
+                    - --max_length_text
+                    - "512"
+                    - --num_workers
+                    - "8"
+                    - --eval_dataset
+                    - --use_wandb
+                    - --wandb_project
+                    - protein-llm-contrastive
+                    - --logging_steps
+                    - "100"
+                    - --eval_steps
+                    - "500"
+                    - --save_steps
+                    - "1000"
+                codePath: wangyujia/BioReason_new/train_contrastive.py
+                codePathLocal: train_contrastive.py
+                cpu_count: 64
+                cpu_count_logical: 64
+                cudaVersion: "12.1"
+                disk:
+                    /:
+                        total: "1623302262784"
+                        used: "1015054336"
+                email: gia0603yucca@gmail.com
+                executable: /root/miniconda3/envs/bioreason/bin/python
+                git:
+                    commit: b8caf406aa1699c788f0ca6e44a1769452c317db
+                    remote: https://github.com/PorUna-byte/PAR.git
+                gpu: NVIDIA A800-SXM4-80GB
+                gpu_count: 8
+                gpu_nvidia:
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-62fcc532-880f-7324-eb13-3b739a38c29b
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-af1197aa-3e7d-a53e-185d-04b1d64164da
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-3e6e551f-2d7d-a27b-654a-da56933d2283
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-a947b64a-c545-ee5e-30e6-b4b0e2dd8fdc
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-ebd04f3f-c437-1b8b-26f2-4b1ebfee93c4
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-53994891-72a6-25be-a4c8-e32c695155b0
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-a927d00a-15c5-31f5-9ae1-9d2e773821a1
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-93d555c4-29a0-cd78-abac-d98cd160d4e1
+                host: dsw-265304-585cc9d768-ckfd7
+                memory:
+                    total: "549755813888"
+                os: Linux-5.10.134-008.18.kangaroo.al8.x86_64-x86_64-with-glibc2.35
+                program: /nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py
+                python: CPython 3.11.0
+                root: /nas/shared/kilab/wangyujia/BioReason_new
+                startedAt: "2025-08-13T06:18:19.380426Z"
+                writerId: 783suewtq8mq4vax242rrsp73rsqcggf
+        m:
+            - "1": train/global_step
+              "6":
+                - 3
+              "7": []
+            - "2": '*'
+              "5": 1
+              "6":
+                - 1
+              "7": []
+        python_version: 3.11.0
+        t:
+            "1":
+                - 1
+                - 9
+                - 11
+                - 41
+                - 49
+                - 51
+                - 71
+                - 84
+                - 98
+                - 103
+            "2":
+                - 1
+                - 9
+                - 11
+                - 41
+                - 49
+                - 51
+                - 71
+                - 84
+                - 98
+                - 103
+            "3":
+                - 7
+                - 13
+                - 16
+                - 66
+            "4": 3.11.0
+            "5": 0.21.1
+            "6": 4.55.0
+            "9":
+                "1": transformers_trainer
+            "12": 0.21.1
+            "13": linux-x86_64
+accelerator_config:
+    value:
+        dispatch_batches: null
+        even_batches: true
+        gradient_accumulation_kwargs: null
+        non_blocking: false
+        split_batches: false
+        use_seedable_sampler: true
+adafactor:
+    value: false
+adam_beta1:
+    value: 0.9
+adam_beta2:
+    value: 0.999
+adam_epsilon:
+    value: 1e-08
+auto_find_batch_size:
+    value: false
+average_tokens_across_devices:
+    value: false
+batch_eval_metrics:
+    value: false
+batch_size:
+    value: 8
+bf16:
+    value: false
+bf16_full_eval:
+    value: false
+cache_dir:
+    value: /model-weights
+data_seed:
+    value: null
+dataloader_drop_last:
+    value: false
+dataloader_num_workers:
+    value: 8
+dataloader_persistent_workers:
+    value: false
+dataloader_pin_memory:
+    value: true
+dataloader_prefetch_factor:
+    value: null
+ddp_backend:
+    value: null
+ddp_broadcast_buffers:
+    value: null
+ddp_bucket_cap_mb:
+    value: null
+ddp_find_unused_parameters:
+    value: false
+ddp_timeout:
+    value: 1800
+debug:
+    value: []
+deepspeed:
+    value: null
+disable_tqdm:
+    value: false
+do_eval:
+    value: true
+do_predict:
+    value: false
+do_train:
+    value: false
+enable_ptm:
+    value: true
+eval_accumulation_steps:
+    value: null
+eval_dataset:
+    value: true
+eval_delay:
+    value: 0
+eval_do_concat_batches:
+    value: true
+eval_on_start:
+    value: false
+eval_steps:
+    value: 500
+eval_strategy:
+    value: steps
+eval_use_gather_object:
+    value: false
+fp16:
+    value: false
+fp16_backend:
+    value: auto
+fp16_full_eval:
+    value: false
+fp16_opt_level:
+    value: O1
+freeze_protein_model:
+    value: true
+freeze_text_model:
+    value: true
+fsdp:
+    value: []
+fsdp_config:
+    value:
+        min_num_params: 0
+        xla: false
+        xla_fsdp_grad_ckpt: false
+        xla_fsdp_v2: false
+fsdp_min_num_params:
+    value: 0
+fsdp_transformer_layer_cls_to_wrap:
+    value: null
+full_determinism:
+    value: false
+gradient_accumulation_steps:
+    value: 1
+gradient_checkpointing:
+    value: false
+gradient_checkpointing_kwargs:
+    value: null
+greater_is_better:
+    value: true
+group_by_length:
+    value: false
+half_precision_backend:
+    value: auto
+hub_always_push:
+    value: false
+hub_model_id:
+    value: null
+hub_private_repo:
+    value: null
+hub_revision:
+    value: null
+hub_strategy:
+    value: every_save
+hub_token:
+    value: <HUB_TOKEN>
+ignore_data_skip:
+    value: false
+include_for_metrics:
+    value: []
+include_inputs_for_metrics:
+    value: false
+include_num_input_tokens_seen:
+    value: false
+include_tokens_per_second:
+    value: false
+jit_mode_eval:
+    value: false
+label_names:
+    value: null
+label_smoothing_factor:
+    value: 0
+learning_rate:
+    value: 0.0001
+length_column_name:
+    value: length
+liger_kernel_config:
+    value: null
+load_best_model_at_end:
+    value: true
+local_rank:
+    value: 0
+log_level:
+    value: passive
+log_level_replica:
+    value: warning
+log_on_each_node:
+    value: true
+logging_dir:
+    value: ./contrastive_outputs/runs/Aug13_14-18-29_dsw-265304-585cc9d768-ckfd7
+logging_first_step:
+    value: false
+logging_nan_inf_filter:
+    value: true
+logging_steps:
+    value: 100
+logging_strategy:
+    value: steps
+lr_scheduler_type:
+    value: linear
+max_grad_norm:
+    value: 1
+max_length_protein:
+    value: 1024
+max_length_text:
+    value: 512
+max_steps:
+    value: -1
+metric_for_best_model:
+    value: eval_avg_recall_at_1
+mp_parameters:
+    value: ""
+neftune_noise_alpha:
+    value: null
+no_cuda:
+    value: false
+num_epochs:
+    value: 10
+num_query_tokens:
+    value: 8
+num_train_epochs:
+    value: 10
+num_workers:
+    value: 8
+optim:
+    value: adamw_torch
+optim_args:
+    value: null
+optim_target_modules:
+    value: null
+output_dir:
+    value: ./contrastive_outputs
+overwrite_output_dir:
+    value: false
+past_index:
+    value: -1
+per_device_eval_batch_size:
+    value: 8
+per_device_train_batch_size:
+    value: 8
+per_gpu_eval_batch_size:
+    value: null
+per_gpu_train_batch_size:
+    value: null
+prediction_loss_only:
+    value: false
+protein_model_name:
+    value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m
+protein_weight:
+    value: 1
+ptm_weight:
+    value: 1
+push_to_hub:
+    value: false
+push_to_hub_model_id:
+    value: null
+push_to_hub_organization:
+    value: null
+push_to_hub_token:
+    value: <PUSH_TO_HUB_TOKEN>
+qformer_model_name:
+    value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft
+ray_scope:
+    value: last
+remove_unused_columns:
+    value: false
+report_to:
+    value:
+        - wandb
+restore_callback_states_from_checkpoint:
+    value: false
+resume_from_checkpoint:
+    value: null
+run_name:
+    value: null
+save_on_each_node:
+    value: false
+save_only_model:
+    value: false
+save_safetensors:
+    value: true
+save_steps:
+    value: 1000
+save_strategy:
+    value: steps
+save_total_limit:
+    value: 3
+seed:
+    value: 42
+skip_memory_metrics:
+    value: true
+temperature:
+    value: 0.07
+text_model_name:
+    value: /oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged
+text_weight:
+    value: 1
+tf32:
+    value: null
+torch_compile:
+    value: false
+torch_compile_backend:
+    value: null
+torch_compile_mode:
+    value: null
+torch_empty_cache_steps:
+    value: null
+torchdynamo:
+    value: null
+tpu_metrics_debug:
+    value: false
+tpu_num_cores:
+    value: null
+train_dataset:
+    value: /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl
+use_cpu:
+    value: false
+use_ipex:
+    value: false
+use_legacy_prediction_loop:
+    value: false
+use_liger_kernel:
+    value: false
+use_mps_device:
+    value: false
+use_wandb:
+    value: true
+valid_dataset:
+    value: /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl
+wandb_entity:
+    value: null
+wandb_project:
+    value: protein-llm-contrastive
+warmup_ratio:
+    value: 0
+warmup_steps:
+    value: 1000
+weight_decay:
+    value: 0.01

BioReason_new/wandb/run-20250813_141819-6d95o7hw/files/output.log

@@ -0,0 +1,45 @@
+Loading model...
+Loading checkpoint shards: 100%|██████████████████████████████████████████████████████| 4/4 [00:01<00:00,  2.48it/s]
+Some weights of EsmModel were not initialized from the model checkpoint at /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m and are newly initialized: ['pooler.dense.bias', 'pooler.dense.weight']
+You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
+Loading datasets...
+Training dataset size: 430595
+Eval dataset size: 10000
+Initializing enhanced trainer...
+[2025-08-13 14:18:33,864] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/_distutils_hack/__init__.py:53: UserWarning: Reliance on distutils from stdlib is deprecated. Users must rely on setuptools to provide the distutils module. Avoid importing distutils or import setuptools first, and avoid setting SETUPTOOLS_USE_DISTUTILS=stdlib. Register concerns at https://github.com/pypa/setuptools/issues/new?template=distutils-deprecation.yml
+  warnings.warn(
+[2025-08-13 14:18:34,947] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False
+Starting enhanced contrastive training...
+- Contrastive learning enabled
+- Protein-text matching: enabled
+- Temperature: 0.07
+- PTM weight: 1.0
+  0%|                                                                                     | 0/67290 [00:00<?, ?it/s]Traceback (most recent call last):
+  File "/nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py", line 552, in <module>
+    trainer = main(args)
+              ^^^^^^^^^^
+  File "/nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py", line 399, in main
+    trainer.train()
+  File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/trainer.py", line 2238, in train
+    return inner_training_loop(
+           ^^^^^^^^^^^^^^^^^^^^
+  File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/trainer.py", line 2582, in _inner_training_loop
+    tr_loss_step = self.training_step(model, inputs, num_items_in_batch)
+                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/trainer.py", line 3796, in training_step
+    loss = self.compute_loss(model, inputs, num_items_in_batch=num_items_in_batch)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/nas/shared/kilab/wangyujia/BioReason_new/bioreason/trainer/contrast_trainer_new.py", line 289, in compute_loss
+    protein_features = model.get_protein_features(protein_sequences)  # [B, num_queries, embed_dim]
+                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/nas/shared/kilab/wangyujia/BioReason_new/bioreason/models/protein_llm.py", line 429, in get_protein_features
+    protein_features = self.protein_projection(
+                       ^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1762, in _call_impl
+    return forward_call(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+TypeError: QFormerProjector.forward() got an unexpected keyword argument 'token_type_ids'

BioReason_new/wandb/run-20250813_141819-6d95o7hw/files/requirements.txt

@@ -0,0 +1,233 @@
+jupyterlab_pygments==0.3.0
+depyf==0.19.0
+av==15.0.0
+fastapi==0.116.1
+ray==2.48.0
+cachetools==6.1.0
+prometheus_client==0.22.1
+requests==2.32.4
+jupyter-lsp==2.2.6
+jupyter-events==0.12.0
+anyio==4.10.0
+pybase64==1.4.2
+fastjsonschema==2.21.1
+cloudpickle==3.1.1
+sentry-sdk==2.34.1
+torchmetrics==1.8.1
+wheel==0.45.1
+idna==3.10
+defusedxml==0.7.1
+starlette==0.47.2
+python-dotenv==1.1.1
+comm==0.2.3
+multidict==6.6.3
+python-json-logger==3.3.0
+jupyter_server==2.16.0
+nvidia-nvtx-cu12==12.6.77
+websockets==15.0.1
+referencing==0.36.2
+gguf==0.17.1
+lm-format-enforcer==0.10.12
+xformers==0.0.31
+ninja==1.11.1.4
+beautifulsoup4==4.13.4
+dnspython==2.7.0
+certifi==2025.8.3
+markdown-it-py==3.0.0
+stack-data==0.6.3
+nvidia-cusparselt-cu12==0.6.3
+networkx==3.5
+aiohappyeyeballs==2.6.1
+nvidia-cuda-cupti-cu12==12.6.80
+llguidance==0.7.30
+rich==14.1.0
+outlines_core==0.2.10
+nvidia-nccl-cu12==2.26.2
+pandas==2.3.1
+pydantic_core==2.33.2
+argon2-cffi==25.1.0
+babel==2.17.0
+isoduration==20.11.0
+uvloop==0.21.0
+json5==0.12.0
+widgetsnbextension==4.0.14
+aiohttp==3.12.15
+notebook_shim==0.2.4
+pandocfilters==1.5.1
+vllm==0.10.0
+blake3==1.0.5
+diskcache==5.6.3
+h11==0.16.0
+attrs==25.3.0
+nvidia-cusolver-cu12==11.7.1.2
+click==8.2.1
+cupy-cuda12x==13.5.1
+jedi==0.19.2
+py-cpuinfo==9.0.0
+overrides==7.7.0
+pytz==2025.2
+protobuf==6.31.1
+dill==0.3.8
+ipykernel==6.30.1
+frozenlist==1.7.0
+huggingface-hub==0.34.4
+rignore==0.6.4
+mistune==3.1.3
+sniffio==1.3.1
+httpx==0.28.1
+sentencepiece==0.2.0
+typing_extensions==4.14.1
+typing-inspection==0.4.1
+async-lru==2.0.5
+bioreason==0.1.0
+charset-normalizer==3.4.3
+llvmlite==0.44.0
+safetensors==0.6.2
+fqdn==1.5.1
+peft==0.17.0
+fastapi-cli==0.0.8
+torch==2.7.1
+jsonschema==4.25.0
+torchvision==0.22.1
+parso==0.8.4
+pillow==11.3.0
+pycountry==24.6.1
+scipy==1.16.1
+wandb==0.21.1
+pip==25.1
+wcwidth==0.2.13
+mistral_common==1.8.3
+nvidia-cudnn-cu12==9.5.1.17
+asttokens==3.0.0
+numpy==2.2.6
+uvicorn==0.35.0
+jupyter_client==8.6.3
+debugpy==1.8.16
+fastapi-cloud-cli==0.1.5
+compressed-tensors==0.10.2
+GitPython==3.1.45
+nvidia-ml-py==13.580.65
+tiktoken==0.11.0
+partial-json-parser==0.2.1.1.post6
+prompt_toolkit==3.0.51
+uri-template==1.3.0
+pyarrow==21.0.0
+nvidia-cuda-nvrtc-cu12==12.6.77
+propcache==0.3.2
+matplotlib-inline==0.1.7
+pure_eval==0.2.3
+nest-asyncio==1.6.0
+deepspeed==0.17.4
+einops==0.8.1
+triton==3.3.1
+trl==0.21.0
+pyzmq==27.0.1
+httptools==0.6.4
+ipython==9.4.0
+packaging==25.0
+hjson==3.1.0
+soxr==0.5.0.post1
+webcolors==24.11.1
+PyYAML==6.0.2
+ipywidgets==8.1.7
+lark==1.2.2
+python-multipart==0.0.20
+tinycss2==1.4.0
+mpmath==1.3.0
+psutil==7.0.0
+nbformat==5.10.4
+qwen-vl-utils==0.0.11
+notebook==7.4.5
+jupyterlab==4.4.5
+transformers==4.55.0
+urllib3==2.5.0
+terminado==0.18.1
+xgrammar==0.1.21
+xxhash==3.5.0
+tzdata==2025.2
+jupyterlab_widgets==3.0.15
+aiosignal==1.4.0
+annotated-types==0.7.0
+nvidia-nvjitlink-cu12==12.6.85
+httpcore==1.0.9
+pexpect==4.9.0
+tornado==6.5.2
+regex==2025.7.33
+filelock==3.18.0
+nvidia-cuda-runtime-cu12==12.6.77
+openai==1.90.0
+Pygments==2.19.2
+cffi==1.17.1
+nvidia-cufft-cu12==11.3.0.4
+rpds-py==0.27.0
+rich-toolkit==0.15.0
+bleach==6.2.0
+sympy==1.14.0
+python-dateutil==2.9.0.post0
+watchfiles==1.1.0
+nbclient==0.10.2
+hf-xet==1.1.7
+Send2Trash==1.8.3
+smmap==5.0.2
+tokenizers==0.21.4
+yarl==1.20.1
+MarkupSafe==3.0.2
+prometheus-fastapi-instrumentator==7.1.0
+rfc3986-validator==0.1.1
+bitsandbytes==0.46.1
+nvidia-cufile-cu12==1.11.1.6
+fsspec==2025.3.0
+executing==2.2.0
+pydantic==2.11.7
+jupyter_core==5.8.1
+msgspec==0.19.0
+nvidia-cusparse-cu12==12.5.4.2
+opencv-python-headless==4.12.0.88
+webencodings==0.5.1
+ptyprocess==0.7.0
+pytorch-lightning==2.5.2
+Jinja2==3.1.6
+websocket-client==1.8.0
+jsonpointer==3.0.0
+traitlets==5.14.3
+numba==0.61.2
+soupsieve==2.7
+cbor2==5.6.5
+multiprocess==0.70.16
+email_validator==2.2.0
+distro==1.9.0
+accelerate==1.10.0
+torchaudio==2.7.1
+jiter==0.10.0
+rfc3339-validator==0.1.4
+jupyter==1.1.1
+typer==0.16.0
+gitdb==4.0.12
+types-python-dateutil==2.9.0.20250809
+soundfile==0.13.1
+astor==0.8.1
+shellingham==1.5.4
+jupyterlab_server==2.27.3
+nvidia-curand-cu12==10.3.7.77
+jupyter-console==6.6.3
+msgpack==1.1.1
+six==1.17.0
+pycparser==2.22
+arrow==1.3.0
+tqdm==4.67.1
+lightning-utilities==0.15.2
+interegular==0.3.3
+rfc3987-syntax==1.1.0
+ipython_pygments_lexers==1.1.1
+datasets==4.0.0
+jupyter_server_terminals==0.5.3
+platformdirs==4.3.8
+pydantic-extra-types==2.10.5
+decorator==5.2.1
+mdurl==0.1.2
+nvidia-cublas-cu12==12.6.4.1
+jsonschema-specifications==2025.4.1
+setuptools==78.1.1
+fastrlock==0.8.3
+argon2-cffi-bindings==25.1.0
+nbconvert==7.16.6

BioReason_new/wandb/run-20250813_141819-6d95o7hw/files/wandb-metadata.json

@@ -0,0 +1,116 @@
+{
+  "os": "Linux-5.10.134-008.18.kangaroo.al8.x86_64-x86_64-with-glibc2.35",
+  "python": "CPython 3.11.0",
+  "startedAt": "2025-08-13T06:18:19.380426Z",
+  "args": [
+    "--text_model_name",
+    "/oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged",
+    "--protein_model_name",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m",
+    "--qformer_model_name",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft",
+    "--num_query_tokens",
+    "8",
+    "--train_dataset",
+    "/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl",
+    "--valid_dataset",
+    "/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl",
+    "--output_dir",
+    "./contrastive_outputs",
+    "--num_epochs",
+    "10",
+    "--batch_size",
+    "8",
+    "--learning_rate",
+    "1e-4",
+    "--temperature",
+    "0.07",
+    "--freeze_protein_model",
+    "--freeze_text_model",
+    "--enable_ptm",
+    "--max_length_protein",
+    "1024",
+    "--max_length_text",
+    "512",
+    "--num_workers",
+    "8",
+    "--eval_dataset",
+    "--use_wandb",
+    "--wandb_project",
+    "protein-llm-contrastive",
+    "--logging_steps",
+    "100",
+    "--eval_steps",
+    "500",
+    "--save_steps",
+    "1000"
+  ],
+  "program": "/nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py",
+  "codePath": "wangyujia/BioReason_new/train_contrastive.py",
+  "codePathLocal": "train_contrastive.py",
+  "git": {
+    "remote": "https://github.com/PorUna-byte/PAR.git",
+    "commit": "b8caf406aa1699c788f0ca6e44a1769452c317db"
+  },
+  "email": "gia0603yucca@gmail.com",
+  "root": "/nas/shared/kilab/wangyujia/BioReason_new",
+  "host": "dsw-265304-585cc9d768-ckfd7",
+  "executable": "/root/miniconda3/envs/bioreason/bin/python",
+  "cpu_count": 64,
+  "cpu_count_logical": 64,
+  "gpu": "NVIDIA A800-SXM4-80GB",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1623302262784",
+      "used": "1015054336"
+    }
+  },
+  "memory": {
+    "total": "549755813888"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-62fcc532-880f-7324-eb13-3b739a38c29b"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-af1197aa-3e7d-a53e-185d-04b1d64164da"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-3e6e551f-2d7d-a27b-654a-da56933d2283"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-a947b64a-c545-ee5e-30e6-b4b0e2dd8fdc"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-ebd04f3f-c437-1b8b-26f2-4b1ebfee93c4"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-53994891-72a6-25be-a4c8-e32c695155b0"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-a927d00a-15c5-31f5-9ae1-9d2e773821a1"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-93d555c4-29a0-cd78-abac-d98cd160d4e1"
+    }
+  ],
+  "cudaVersion": "12.1",
+  "writerId": "783suewtq8mq4vax242rrsp73rsqcggf"
+}

BioReason_new/wandb/run-20250813_141819-6d95o7hw/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_runtime":14,"_wandb":{"runtime":14}}

BioReason_new/wandb/run-20250813_141819-6d95o7hw/logs/debug-internal.log

@@ -0,0 +1,19 @@
+{"time":"2025-08-13T14:18:19.679275905+08:00","level":"INFO","msg":"stream: starting","core version":"0.21.1"}
+{"time":"2025-08-13T14:18:23.516936114+08:00","level":"INFO","msg":"stream: created new stream","id":"6d95o7hw"}
+{"time":"2025-08-13T14:18:23.517759676+08:00","level":"INFO","msg":"stream: started","id":"6d95o7hw"}
+{"time":"2025-08-13T14:18:23.51779428+08:00","level":"INFO","msg":"sender: started","stream_id":"6d95o7hw"}
+{"time":"2025-08-13T14:18:23.517793771+08:00","level":"INFO","msg":"writer: started","stream_id":"6d95o7hw"}
+{"time":"2025-08-13T14:18:23.51783624+08:00","level":"INFO","msg":"handler: started","stream_id":"6d95o7hw"}
+{"time":"2025-08-13T14:18:26.306463347+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/6d95o7hw/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T061824Z&X-Goog-Expires=86399&X-Goog-Signature=368653d66d2fa0c72af20425484a97b74c80c623cfbcd113e7638d58e672856ed1719e829d5fc48438e50516e1a081ff4ee135505f606a06aadfda935ac087e929f22e6edc07e8cf86a943206508b5d114de7866e3e0821e24b7ee872f36b33aa05d0717c89256355d27798137cf600bb5e7c6aa36fabb3ff1a067817560ee05597e6a1d83e088f9158e3280c52aa2bbc1206d9ac932d20a877c3b213397a2da7705231efec42ce78cfcd3d69537d6da25e05fead16f6edb6da6c14b855e35c462e711eb2b6b0edc20eb6bd5d253e99ae0f83dbbac9fb66398eb9c28cfa64f0a4cda60517b87b93ca858d27ff7945af913737b955d2f49799747cbdb2beedc86&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:36176->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/6d95o7hw/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T061824Z&X-Goog-Expires=86399&X-Goog-Signature=368653d66d2fa0c72af20425484a97b74c80c623cfbcd113e7638d58e672856ed1719e829d5fc48438e50516e1a081ff4ee135505f606a06aadfda935ac087e929f22e6edc07e8cf86a943206508b5d114de7866e3e0821e24b7ee872f36b33aa05d0717c89256355d27798137cf600bb5e7c6aa36fabb3ff1a067817560ee05597e6a1d83e088f9158e3280c52aa2bbc1206d9ac932d20a877c3b213397a2da7705231efec42ce78cfcd3d69537d6da25e05fead16f6edb6da6c14b855e35c462e711eb2b6b0edc20eb6bd5d253e99ae0f83dbbac9fb66398eb9c28cfa64f0a4cda60517b87b93ca858d27ff7945af913737b955d2f49799747cbdb2beedc86&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:18:28.860760307+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/6d95o7hw/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T061824Z&X-Goog-Expires=86399&X-Goog-Signature=368653d66d2fa0c72af20425484a97b74c80c623cfbcd113e7638d58e672856ed1719e829d5fc48438e50516e1a081ff4ee135505f606a06aadfda935ac087e929f22e6edc07e8cf86a943206508b5d114de7866e3e0821e24b7ee872f36b33aa05d0717c89256355d27798137cf600bb5e7c6aa36fabb3ff1a067817560ee05597e6a1d83e088f9158e3280c52aa2bbc1206d9ac932d20a877c3b213397a2da7705231efec42ce78cfcd3d69537d6da25e05fead16f6edb6da6c14b855e35c462e711eb2b6b0edc20eb6bd5d253e99ae0f83dbbac9fb66398eb9c28cfa64f0a4cda60517b87b93ca858d27ff7945af913737b955d2f49799747cbdb2beedc86&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:40306->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/6d95o7hw/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T061824Z&X-Goog-Expires=86399&X-Goog-Signature=368653d66d2fa0c72af20425484a97b74c80c623cfbcd113e7638d58e672856ed1719e829d5fc48438e50516e1a081ff4ee135505f606a06aadfda935ac087e929f22e6edc07e8cf86a943206508b5d114de7866e3e0821e24b7ee872f36b33aa05d0717c89256355d27798137cf600bb5e7c6aa36fabb3ff1a067817560ee05597e6a1d83e088f9158e3280c52aa2bbc1206d9ac932d20a877c3b213397a2da7705231efec42ce78cfcd3d69537d6da25e05fead16f6edb6da6c14b855e35c462e711eb2b6b0edc20eb6bd5d253e99ae0f83dbbac9fb66398eb9c28cfa64f0a4cda60517b87b93ca858d27ff7945af913737b955d2f49799747cbdb2beedc86&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:18:38.597075766+08:00","level":"INFO","msg":"stream: closing","id":"6d95o7hw"}
+{"time":"2025-08-13T14:18:41.124384983+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/6d95o7hw/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T061839Z&X-Goog-Expires=86399&X-Goog-Signature=37097f0467c2cb0a73f1abeca04122e4d29400711c0c68032c874c811c7fa32f184c9fd288abcafe5538f6df714fd4cdb57a800bca43ecedea9a4987953524ae63497f2c776218d810da53e4e33d6409c5a8152fab28442bafa237041760663a06eb98db75dba3b2f3040076a3abec723cf6dd7c95596ce0915c97d3798df573dafb78c1be90d571173c09d212fda585a3414e5828a4a5d3ee5716d162d784a3954b4d0aa8651a28fbb9864b92a207f9cd6641a23ac16f9a5d4c7a50c5965b353e3bf0d4b1a86f00c37d1530e2f9be58229477567871758828e816cbaf30d2fa684d5f00818dda963017052f6bf7e03fb68c5b195a3a69731b1d9282dbe7cddc&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:57006->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/6d95o7hw/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T061839Z&X-Goog-Expires=86399&X-Goog-Signature=37097f0467c2cb0a73f1abeca04122e4d29400711c0c68032c874c811c7fa32f184c9fd288abcafe5538f6df714fd4cdb57a800bca43ecedea9a4987953524ae63497f2c776218d810da53e4e33d6409c5a8152fab28442bafa237041760663a06eb98db75dba3b2f3040076a3abec723cf6dd7c95596ce0915c97d3798df573dafb78c1be90d571173c09d212fda585a3414e5828a4a5d3ee5716d162d784a3954b4d0aa8651a28fbb9864b92a207f9cd6641a23ac16f9a5d4c7a50c5965b353e3bf0d4b1a86f00c37d1530e2f9be58229477567871758828e816cbaf30d2fa684d5f00818dda963017052f6bf7e03fb68c5b195a3a69731b1d9282dbe7cddc&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:18:42.124134987+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/6d95o7hw/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T061841Z&X-Goog-Expires=86399&X-Goog-Signature=2815ea37d9b1b097f525654a380c6b87428c3213cde6b7197afb098b7d6cbe2a091a67d76b46a30007b5d4a3ae985d776df9d743d51f8b986f8b6e3d1f8648d6cabf09d467065efcfcbc980f395e13a09f3bb3582c483c46b7507551843641ab9a4b695f6fa50b444da9a00a98f6cedc243737a78dc259f25954f50cb54281f837ba898d97069de2bf41f83108e40c4c52980da614fa20f8799fd6474a5445edc9c2b8b593316d25e2ed2ae631b0c39a00f7a5ec0013775ff7e2a32466fa3f935260dc683d0b0acecf7a2250b3aaf031a15c91b9eba92a548979f2e3857c4e1d5f2b2607bb0d909ca9b1d5f759465cc16cd507c7a828a86e771de94e681bd8c4&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:54320->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/6d95o7hw/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T061841Z&X-Goog-Expires=86399&X-Goog-Signature=2815ea37d9b1b097f525654a380c6b87428c3213cde6b7197afb098b7d6cbe2a091a67d76b46a30007b5d4a3ae985d776df9d743d51f8b986f8b6e3d1f8648d6cabf09d467065efcfcbc980f395e13a09f3bb3582c483c46b7507551843641ab9a4b695f6fa50b444da9a00a98f6cedc243737a78dc259f25954f50cb54281f837ba898d97069de2bf41f83108e40c4c52980da614fa20f8799fd6474a5445edc9c2b8b593316d25e2ed2ae631b0c39a00f7a5ec0013775ff7e2a32466fa3f935260dc683d0b0acecf7a2250b3aaf031a15c91b9eba92a548979f2e3857c4e1d5f2b2607bb0d909ca9b1d5f759465cc16cd507c7a828a86e771de94e681bd8c4&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:20:35.777590448+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/6d95o7hw/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T061824Z&X-Goog-Expires=86399&X-Goog-Signature=a4e383bc3740704387f69b5cfd695ead4a69bfe155cc9a50a50eb3ca73735af4d51452984ffe7eb753b3c1c6aa27268975caeb326500d7a5227fe2dbcb9b77f36af4c8a84e285d9679374a7ac036b8565fdac7f706da3cc647b019a8838ae52d1bfb7ecc691c2c3aebd5c3df1d894a6f2a299de849704ad52c1f89f1d7a0a607b7a800e6caac3e331eaa671b38cd87a45c18987fdfce0f65642b8c94fc29fa3222c28e5faf610d144cfad1931f65d3a139551332bcf950b5c7e6dd5a4c29c3fa8469b7f532aa53837a1bc8764b48c318b3af9606fcf4de722e081469dfe99c34c35dd9e563637aa31ca3923ad22535e27c24d6ab52bcb8451026bc37e4bcbe23&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:58284->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/6d95o7hw/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T061824Z&X-Goog-Expires=86399&X-Goog-Signature=a4e383bc3740704387f69b5cfd695ead4a69bfe155cc9a50a50eb3ca73735af4d51452984ffe7eb753b3c1c6aa27268975caeb326500d7a5227fe2dbcb9b77f36af4c8a84e285d9679374a7ac036b8565fdac7f706da3cc647b019a8838ae52d1bfb7ecc691c2c3aebd5c3df1d894a6f2a299de849704ad52c1f89f1d7a0a607b7a800e6caac3e331eaa671b38cd87a45c18987fdfce0f65642b8c94fc29fa3222c28e5faf610d144cfad1931f65d3a139551332bcf950b5c7e6dd5a4c29c3fa8469b7f532aa53837a1bc8764b48c318b3af9606fcf4de722e081469dfe99c34c35dd9e563637aa31ca3923ad22535e27c24d6ab52bcb8451026bc37e4bcbe23&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:20:56.270453021+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/6d95o7hw/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T061841Z&X-Goog-Expires=86399&X-Goog-Signature=2815ea37d9b1b097f525654a380c6b87428c3213cde6b7197afb098b7d6cbe2a091a67d76b46a30007b5d4a3ae985d776df9d743d51f8b986f8b6e3d1f8648d6cabf09d467065efcfcbc980f395e13a09f3bb3582c483c46b7507551843641ab9a4b695f6fa50b444da9a00a98f6cedc243737a78dc259f25954f50cb54281f837ba898d97069de2bf41f83108e40c4c52980da614fa20f8799fd6474a5445edc9c2b8b593316d25e2ed2ae631b0c39a00f7a5ec0013775ff7e2a32466fa3f935260dc683d0b0acecf7a2250b3aaf031a15c91b9eba92a548979f2e3857c4e1d5f2b2607bb0d909ca9b1d5f759465cc16cd507c7a828a86e771de94e681bd8c4&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:37190->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/6d95o7hw/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T061841Z&X-Goog-Expires=86399&X-Goog-Signature=2815ea37d9b1b097f525654a380c6b87428c3213cde6b7197afb098b7d6cbe2a091a67d76b46a30007b5d4a3ae985d776df9d743d51f8b986f8b6e3d1f8648d6cabf09d467065efcfcbc980f395e13a09f3bb3582c483c46b7507551843641ab9a4b695f6fa50b444da9a00a98f6cedc243737a78dc259f25954f50cb54281f837ba898d97069de2bf41f83108e40c4c52980da614fa20f8799fd6474a5445edc9c2b8b593316d25e2ed2ae631b0c39a00f7a5ec0013775ff7e2a32466fa3f935260dc683d0b0acecf7a2250b3aaf031a15c91b9eba92a548979f2e3857c4e1d5f2b2607bb0d909ca9b1d5f759465cc16cd507c7a828a86e771de94e681bd8c4&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:21:00.865375075+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/6d95o7hw/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T061841Z&X-Goog-Expires=86399&X-Goog-Signature=2815ea37d9b1b097f525654a380c6b87428c3213cde6b7197afb098b7d6cbe2a091a67d76b46a30007b5d4a3ae985d776df9d743d51f8b986f8b6e3d1f8648d6cabf09d467065efcfcbc980f395e13a09f3bb3582c483c46b7507551843641ab9a4b695f6fa50b444da9a00a98f6cedc243737a78dc259f25954f50cb54281f837ba898d97069de2bf41f83108e40c4c52980da614fa20f8799fd6474a5445edc9c2b8b593316d25e2ed2ae631b0c39a00f7a5ec0013775ff7e2a32466fa3f935260dc683d0b0acecf7a2250b3aaf031a15c91b9eba92a548979f2e3857c4e1d5f2b2607bb0d909ca9b1d5f759465cc16cd507c7a828a86e771de94e681bd8c4&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:34918->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/6d95o7hw/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T061841Z&X-Goog-Expires=86399&X-Goog-Signature=2815ea37d9b1b097f525654a380c6b87428c3213cde6b7197afb098b7d6cbe2a091a67d76b46a30007b5d4a3ae985d776df9d743d51f8b986f8b6e3d1f8648d6cabf09d467065efcfcbc980f395e13a09f3bb3582c483c46b7507551843641ab9a4b695f6fa50b444da9a00a98f6cedc243737a78dc259f25954f50cb54281f837ba898d97069de2bf41f83108e40c4c52980da614fa20f8799fd6474a5445edc9c2b8b593316d25e2ed2ae631b0c39a00f7a5ec0013775ff7e2a32466fa3f935260dc683d0b0acecf7a2250b3aaf031a15c91b9eba92a548979f2e3857c4e1d5f2b2607bb0d909ca9b1d5f759465cc16cd507c7a828a86e771de94e681bd8c4&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:22:46.942241349+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/6d95o7hw/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T061824Z&X-Goog-Expires=86399&X-Goog-Signature=a4e383bc3740704387f69b5cfd695ead4a69bfe155cc9a50a50eb3ca73735af4d51452984ffe7eb753b3c1c6aa27268975caeb326500d7a5227fe2dbcb9b77f36af4c8a84e285d9679374a7ac036b8565fdac7f706da3cc647b019a8838ae52d1bfb7ecc691c2c3aebd5c3df1d894a6f2a299de849704ad52c1f89f1d7a0a607b7a800e6caac3e331eaa671b38cd87a45c18987fdfce0f65642b8c94fc29fa3222c28e5faf610d144cfad1931f65d3a139551332bcf950b5c7e6dd5a4c29c3fa8469b7f532aa53837a1bc8764b48c318b3af9606fcf4de722e081469dfe99c34c35dd9e563637aa31ca3923ad22535e27c24d6ab52bcb8451026bc37e4bcbe23&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:36662->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/6d95o7hw/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T061824Z&X-Goog-Expires=86399&X-Goog-Signature=a4e383bc3740704387f69b5cfd695ead4a69bfe155cc9a50a50eb3ca73735af4d51452984ffe7eb753b3c1c6aa27268975caeb326500d7a5227fe2dbcb9b77f36af4c8a84e285d9679374a7ac036b8565fdac7f706da3cc647b019a8838ae52d1bfb7ecc691c2c3aebd5c3df1d894a6f2a299de849704ad52c1f89f1d7a0a607b7a800e6caac3e331eaa671b38cd87a45c18987fdfce0f65642b8c94fc29fa3222c28e5faf610d144cfad1931f65d3a139551332bcf950b5c7e6dd5a4c29c3fa8469b7f532aa53837a1bc8764b48c318b3af9606fcf4de722e081469dfe99c34c35dd9e563637aa31ca3923ad22535e27c24d6ab52bcb8451026bc37e4bcbe23&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:22:53.1515727+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-08-13T14:23:06.271059888+08:00","level":"INFO","msg":"handler: closed","stream_id":"6d95o7hw"}
+{"time":"2025-08-13T14:23:06.276503172+08:00","level":"INFO","msg":"sender: closed","stream_id":"6d95o7hw"}
+{"time":"2025-08-13T14:23:06.276536784+08:00","level":"INFO","msg":"stream: closed","id":"6d95o7hw"}

BioReason_new/wandb/run-20250813_141819-6d95o7hw/logs/debug.log

@@ -0,0 +1,23 @@
+2025-08-13 14:18:19,458 INFO    MainThread:7754 [wandb_setup.py:_flush():80] Current SDK version is 0.21.1
+2025-08-13 14:18:19,458 INFO    MainThread:7754 [wandb_setup.py:_flush():80] Configure stats pid to 7754
+2025-08-13 14:18:19,458 INFO    MainThread:7754 [wandb_setup.py:_flush():80] Loading settings from /root/.config/wandb/settings
+2025-08-13 14:18:19,458 INFO    MainThread:7754 [wandb_setup.py:_flush():80] Loading settings from /nas/shared/kilab/wangyujia/BioReason_new/wandb/settings
+2025-08-13 14:18:19,458 INFO    MainThread:7754 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-08-13 14:18:19,458 INFO    MainThread:7754 [wandb_init.py:setup_run_log_directory():703] Logging user logs to /nas/shared/kilab/wangyujia/BioReason_new/wandb/run-20250813_141819-6d95o7hw/logs/debug.log
+2025-08-13 14:18:19,458 INFO    MainThread:7754 [wandb_init.py:setup_run_log_directory():704] Logging internal logs to /nas/shared/kilab/wangyujia/BioReason_new/wandb/run-20250813_141819-6d95o7hw/logs/debug-internal.log
+2025-08-13 14:18:19,458 INFO    MainThread:7754 [wandb_init.py:init():830] calling init triggers
+2025-08-13 14:18:19,458 INFO    MainThread:7754 [wandb_init.py:init():835] wandb.init called with sweep_config: {}
+config: {'text_model_name': '/oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged', 'protein_model_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m', 'qformer_model_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft', 'cache_dir': '/model-weights', 'num_query_tokens': 8, 'train_dataset': '/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl', 'valid_dataset': '/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl', 'eval_dataset': True, 'output_dir': './contrastive_outputs', 'num_epochs': 10, 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 0.01, 'warmup_steps': 1000, 'gradient_accumulation_steps': 1, 'temperature': 0.07, 'freeze_protein_model': True, 'freeze_text_model': True, 'protein_weight': 1.0, 'text_weight': 1.0, 'enable_ptm': True, 'ptm_weight': 1.0, 'max_length_protein': 1024, 'max_length_text': 512, 'num_workers': 8, 'logging_steps': 100, 'eval_steps': 500, 'save_steps': 1000, 'save_total_limit': 3, 'fp16': False, 'bf16': False, 'seed': 42, 'use_wandb': True, 'wandb_project': 'protein-llm-contrastive', 'wandb_entity': None, '_wandb': {}}
+2025-08-13 14:18:19,458 INFO    MainThread:7754 [wandb_init.py:init():871] starting backend
+2025-08-13 14:18:19,667 INFO    MainThread:7754 [wandb_init.py:init():874] sending inform_init request
+2025-08-13 14:18:19,673 INFO    MainThread:7754 [wandb_init.py:init():882] backend started and connected
+2025-08-13 14:18:19,674 INFO    MainThread:7754 [wandb_init.py:init():953] updated telemetry
+2025-08-13 14:18:19,700 INFO    MainThread:7754 [wandb_init.py:init():977] communicating run to backend with 90.0 second timeout
+2025-08-13 14:18:24,474 INFO    MainThread:7754 [wandb_init.py:init():1029] starting run threads in backend
+2025-08-13 14:18:24,583 INFO    MainThread:7754 [wandb_run.py:_console_start():2494] atexit reg
+2025-08-13 14:18:24,583 INFO    MainThread:7754 [wandb_run.py:_redirect():2342] redirect: wrap_raw
+2025-08-13 14:18:24,584 INFO    MainThread:7754 [wandb_run.py:_redirect():2411] Wrapping output streams.
+2025-08-13 14:18:24,584 INFO    MainThread:7754 [wandb_run.py:_redirect():2434] Redirects installed.
+2025-08-13 14:18:24,586 INFO    MainThread:7754 [wandb_init.py:init():1075] run started, returning control to user process
+2025-08-13 14:18:35,224 INFO    MainThread:7754 [wandb_run.py:_config_callback():1380] config_cb None None {'output_dir': './contrastive_outputs', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': True, 'do_predict': False, 'eval_strategy': 'steps', 'prediction_loss_only': False, 'per_device_train_batch_size': 8, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'eval_delay': 0, 'torch_empty_cache_steps': None, 'learning_rate': 0.0001, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 10, 'max_steps': -1, 'lr_scheduler_type': 'linear', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.0, 'warmup_steps': 1000, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': './contrastive_outputs/runs/Aug13_14-18-29_dsw-265304-585cc9d768-ckfd7', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 100, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 1000, 'save_total_limit': 3, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'restore_callback_states_from_checkpoint': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 42, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': False, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 500, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': None, 'disable_tqdm': False, 'remove_unused_columns': False, 'label_names': None, 'load_best_model_at_end': True, 'metric_for_best_model': 'eval_avg_recall_at_1', 'greater_is_better': True, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'deepspeed': None, 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['wandb'], 'ddp_find_unused_parameters': False, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_always_push': False, 'hub_revision': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'include_for_metrics': [], 'eval_do_concat_batches': True, 'fp16_backend': 'auto', 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None, 'batch_eval_metrics': False, 'eval_on_start': False, 'use_liger_kernel': False, 'liger_kernel_config': None, 'eval_use_gather_object': False, 'average_tokens_across_devices': False, 'temperature': 0.07, 'freeze_protein_model': True, 'freeze_text_model': True, 'protein_weight': 1.0, 'text_weight': 1.0, 'max_length_protein': 1024, 'max_length_text': 512, 'enable_ptm': True, 'ptm_weight': 1.0}
+2025-08-13 14:18:38,596 INFO    MsgRouterThr:7754 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles.

BioReason_new/wandb/run-20250813_142935-iunrz6lf/files/config.yaml

@@ -0,0 +1,200 @@
+_wandb:
+    value:
+        cli_version: 0.21.1
+        e:
+            4obixt2d5ipbxfjgs2k7zb94y9cqikyp:
+                args:
+                    - --text_model_name
+                    - /oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged
+                    - --protein_model_name
+                    - /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m
+                    - --qformer_model_name
+                    - /nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft
+                    - --num_query_tokens
+                    - "8"
+                    - --train_dataset
+                    - /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl
+                    - --valid_dataset
+                    - /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl
+                    - --output_dir
+                    - ./contrastive_outputs
+                    - --num_epochs
+                    - "10"
+                    - --batch_size
+                    - "8"
+                    - --learning_rate
+                    - "1e-4"
+                    - --temperature
+                    - "0.07"
+                    - --freeze_protein_model
+                    - --freeze_text_model
+                    - --enable_ptm
+                    - --max_length_protein
+                    - "1024"
+                    - --max_length_text
+                    - "512"
+                    - --num_workers
+                    - "8"
+                    - --eval_dataset
+                    - --use_wandb
+                    - --wandb_project
+                    - protein-llm-contrastive
+                    - --logging_steps
+                    - "100"
+                    - --eval_steps
+                    - "500"
+                    - --save_steps
+                    - "1000"
+                codePath: wangyujia/BioReason_new/train_contrastive.py
+                codePathLocal: train_contrastive.py
+                cpu_count: 64
+                cpu_count_logical: 64
+                cudaVersion: "12.1"
+                disk:
+                    /:
+                        total: "1623302262784"
+                        used: "1015083008"
+                email: gia0603yucca@gmail.com
+                executable: /root/miniconda3/envs/bioreason/bin/python
+                git:
+                    commit: b8caf406aa1699c788f0ca6e44a1769452c317db
+                    remote: https://github.com/PorUna-byte/PAR.git
+                gpu: NVIDIA A800-SXM4-80GB
+                gpu_count: 8
+                gpu_nvidia:
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-62fcc532-880f-7324-eb13-3b739a38c29b
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-af1197aa-3e7d-a53e-185d-04b1d64164da
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-3e6e551f-2d7d-a27b-654a-da56933d2283
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-a947b64a-c545-ee5e-30e6-b4b0e2dd8fdc
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-ebd04f3f-c437-1b8b-26f2-4b1ebfee93c4
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-53994891-72a6-25be-a4c8-e32c695155b0
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-a927d00a-15c5-31f5-9ae1-9d2e773821a1
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-93d555c4-29a0-cd78-abac-d98cd160d4e1
+                host: dsw-265304-585cc9d768-ckfd7
+                memory:
+                    total: "549755813888"
+                os: Linux-5.10.134-008.18.kangaroo.al8.x86_64-x86_64-with-glibc2.35
+                program: /nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py
+                python: CPython 3.11.0
+                root: /nas/shared/kilab/wangyujia/BioReason_new
+                startedAt: "2025-08-13T06:29:35.069039Z"
+                writerId: 4obixt2d5ipbxfjgs2k7zb94y9cqikyp
+        m: []
+        python_version: 3.11.0
+        t:
+            "1":
+                - 1
+                - 9
+                - 11
+                - 41
+                - 49
+                - 51
+                - 71
+                - 84
+                - 98
+                - 103
+            "2":
+                - 1
+                - 9
+                - 11
+                - 41
+                - 49
+                - 51
+                - 71
+                - 84
+                - 98
+                - 103
+            "3":
+                - 13
+                - 16
+            "4": 3.11.0
+            "5": 0.21.1
+            "6": 4.55.0
+            "12": 0.21.1
+            "13": linux-x86_64
+batch_size:
+    value: 8
+bf16:
+    value: false
+cache_dir:
+    value: /model-weights
+enable_ptm:
+    value: true
+eval_dataset:
+    value: true
+eval_steps:
+    value: 500
+fp16:
+    value: false
+freeze_protein_model:
+    value: true
+freeze_text_model:
+    value: true
+gradient_accumulation_steps:
+    value: 1
+learning_rate:
+    value: 0.0001
+logging_steps:
+    value: 100
+max_length_protein:
+    value: 1024
+max_length_text:
+    value: 512
+num_epochs:
+    value: 10
+num_query_tokens:
+    value: 8
+num_workers:
+    value: 8
+output_dir:
+    value: ./contrastive_outputs
+protein_model_name:
+    value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m
+protein_weight:
+    value: 1
+ptm_weight:
+    value: 1
+qformer_model_name:
+    value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft
+save_steps:
+    value: 1000
+save_total_limit:
+    value: 3
+seed:
+    value: 42
+temperature:
+    value: 0.07
+text_model_name:
+    value: /oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged
+text_weight:
+    value: 1
+train_dataset:
+    value: /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl
+use_wandb:
+    value: true
+valid_dataset:
+    value: /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl
+wandb_entity:
+    value: null
+wandb_project:
+    value: protein-llm-contrastive
+warmup_steps:
+    value: 1000
+weight_decay:
+    value: 0.01

BioReason_new/wandb/run-20250813_142935-iunrz6lf/files/output.log

@@ -0,0 +1,40 @@
+Loading model...
+Loading checkpoint shards: 100%|██████████████████████████████████████████████████████| 4/4 [00:01<00:00,  2.64it/s]
+Some weights of EsmModel were not initialized from the model checkpoint at /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m and are newly initialized: ['pooler.dense.bias', 'pooler.dense.weight']
+You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
+Traceback (most recent call last):
+  File "/nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py", line 552, in <module>
+    trainer = main(args)
+              ^^^^^^^^^^
+  File "/nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py", line 303, in main
+    model = ProteinLLMModel(
+            ^^^^^^^^^^^^^^^^
+  File "/nas/shared/kilab/wangyujia/BioReason_new/bioreason/models/protein_llm.py", line 290, in __init__
+    self.protein_projection = QFormerProjector(
+                              ^^^^^^^^^^^^^^^^^
+  File "/nas/shared/kilab/wangyujia/BioReason_new/bioreason/models/protein_llm.py", line 136, in __init__
+    self.qformer = BertModel.from_pretrained(qformer_model_name, config=config)
+                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/modeling_utils.py", line 316, in _wrapper
+    return func(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^
+  File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/modeling_utils.py", line 5061, in from_pretrained
+    ) = cls._load_pretrained_model(
+        ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/modeling_utils.py", line 5524, in _load_pretrained_model
+    _error_msgs, disk_offload_index, cpu_offload_index = load_shard_file(args)
+                                                         ^^^^^^^^^^^^^^^^^^^^^
+  File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/modeling_utils.py", line 974, in load_shard_file
+    disk_offload_index, cpu_offload_index = _load_state_dict_into_meta_model(
+                                            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context
+    return func(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^
+  File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/modeling_utils.py", line 879, in _load_state_dict_into_meta_model
+    _load_parameter_into_model(model, param_name, param.to(param_device))
+  File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/modeling_utils.py", line 739, in _load_parameter_into_model
+    module.load_state_dict({param_type: tensor}, strict=False, assign=True)
+  File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/torch/nn/modules/module.py", line 2593, in load_state_dict
+    raise RuntimeError(
+RuntimeError: Error(s) in loading state_dict for Embedding:
+	size mismatch for weight: copying a param with shape torch.Size([512, 768]) from checkpoint, the shape in current model is torch.Size([1024, 768]).

BioReason_new/wandb/run-20250813_142935-iunrz6lf/files/requirements.txt

@@ -0,0 +1,233 @@
+jupyterlab_pygments==0.3.0
+depyf==0.19.0
+av==15.0.0
+fastapi==0.116.1
+ray==2.48.0
+cachetools==6.1.0
+prometheus_client==0.22.1
+requests==2.32.4
+jupyter-lsp==2.2.6
+jupyter-events==0.12.0
+anyio==4.10.0
+pybase64==1.4.2
+fastjsonschema==2.21.1
+cloudpickle==3.1.1
+sentry-sdk==2.34.1
+torchmetrics==1.8.1
+wheel==0.45.1
+idna==3.10
+defusedxml==0.7.1
+starlette==0.47.2
+python-dotenv==1.1.1
+comm==0.2.3
+multidict==6.6.3
+python-json-logger==3.3.0
+jupyter_server==2.16.0
+nvidia-nvtx-cu12==12.6.77
+websockets==15.0.1
+referencing==0.36.2
+gguf==0.17.1
+lm-format-enforcer==0.10.12
+xformers==0.0.31
+ninja==1.11.1.4
+beautifulsoup4==4.13.4
+dnspython==2.7.0
+certifi==2025.8.3
+markdown-it-py==3.0.0
+stack-data==0.6.3
+nvidia-cusparselt-cu12==0.6.3
+networkx==3.5
+aiohappyeyeballs==2.6.1
+nvidia-cuda-cupti-cu12==12.6.80
+llguidance==0.7.30
+rich==14.1.0
+outlines_core==0.2.10
+nvidia-nccl-cu12==2.26.2
+pandas==2.3.1
+pydantic_core==2.33.2
+argon2-cffi==25.1.0
+babel==2.17.0
+isoduration==20.11.0
+uvloop==0.21.0
+json5==0.12.0
+widgetsnbextension==4.0.14
+aiohttp==3.12.15
+notebook_shim==0.2.4
+pandocfilters==1.5.1
+vllm==0.10.0
+blake3==1.0.5
+diskcache==5.6.3
+h11==0.16.0
+attrs==25.3.0
+nvidia-cusolver-cu12==11.7.1.2
+click==8.2.1
+cupy-cuda12x==13.5.1
+jedi==0.19.2
+py-cpuinfo==9.0.0
+overrides==7.7.0
+pytz==2025.2
+protobuf==6.31.1
+dill==0.3.8
+ipykernel==6.30.1
+frozenlist==1.7.0
+huggingface-hub==0.34.4
+rignore==0.6.4
+mistune==3.1.3
+sniffio==1.3.1
+httpx==0.28.1
+sentencepiece==0.2.0
+typing_extensions==4.14.1
+typing-inspection==0.4.1
+async-lru==2.0.5
+bioreason==0.1.0
+charset-normalizer==3.4.3
+llvmlite==0.44.0
+safetensors==0.6.2
+fqdn==1.5.1
+peft==0.17.0
+fastapi-cli==0.0.8
+torch==2.7.1
+jsonschema==4.25.0
+torchvision==0.22.1
+parso==0.8.4
+pillow==11.3.0
+pycountry==24.6.1
+scipy==1.16.1
+wandb==0.21.1
+pip==25.1
+wcwidth==0.2.13
+mistral_common==1.8.3
+nvidia-cudnn-cu12==9.5.1.17
+asttokens==3.0.0
+numpy==2.2.6
+uvicorn==0.35.0
+jupyter_client==8.6.3
+debugpy==1.8.16
+fastapi-cloud-cli==0.1.5
+compressed-tensors==0.10.2
+GitPython==3.1.45
+nvidia-ml-py==13.580.65
+tiktoken==0.11.0
+partial-json-parser==0.2.1.1.post6
+prompt_toolkit==3.0.51
+uri-template==1.3.0
+pyarrow==21.0.0
+nvidia-cuda-nvrtc-cu12==12.6.77
+propcache==0.3.2
+matplotlib-inline==0.1.7
+pure_eval==0.2.3
+nest-asyncio==1.6.0
+deepspeed==0.17.4
+einops==0.8.1
+triton==3.3.1
+trl==0.21.0
+pyzmq==27.0.1
+httptools==0.6.4
+ipython==9.4.0
+packaging==25.0
+hjson==3.1.0
+soxr==0.5.0.post1
+webcolors==24.11.1
+PyYAML==6.0.2
+ipywidgets==8.1.7
+lark==1.2.2
+python-multipart==0.0.20
+tinycss2==1.4.0
+mpmath==1.3.0
+psutil==7.0.0
+nbformat==5.10.4
+qwen-vl-utils==0.0.11
+notebook==7.4.5
+jupyterlab==4.4.5
+transformers==4.55.0
+urllib3==2.5.0
+terminado==0.18.1
+xgrammar==0.1.21
+xxhash==3.5.0
+tzdata==2025.2
+jupyterlab_widgets==3.0.15
+aiosignal==1.4.0
+annotated-types==0.7.0
+nvidia-nvjitlink-cu12==12.6.85
+httpcore==1.0.9
+pexpect==4.9.0
+tornado==6.5.2
+regex==2025.7.33
+filelock==3.18.0
+nvidia-cuda-runtime-cu12==12.6.77
+openai==1.90.0
+Pygments==2.19.2
+cffi==1.17.1
+nvidia-cufft-cu12==11.3.0.4
+rpds-py==0.27.0
+rich-toolkit==0.15.0
+bleach==6.2.0
+sympy==1.14.0
+python-dateutil==2.9.0.post0
+watchfiles==1.1.0
+nbclient==0.10.2
+hf-xet==1.1.7
+Send2Trash==1.8.3
+smmap==5.0.2
+tokenizers==0.21.4
+yarl==1.20.1
+MarkupSafe==3.0.2
+prometheus-fastapi-instrumentator==7.1.0
+rfc3986-validator==0.1.1
+bitsandbytes==0.46.1
+nvidia-cufile-cu12==1.11.1.6
+fsspec==2025.3.0
+executing==2.2.0
+pydantic==2.11.7
+jupyter_core==5.8.1
+msgspec==0.19.0
+nvidia-cusparse-cu12==12.5.4.2
+opencv-python-headless==4.12.0.88
+webencodings==0.5.1
+ptyprocess==0.7.0
+pytorch-lightning==2.5.2
+Jinja2==3.1.6
+websocket-client==1.8.0
+jsonpointer==3.0.0
+traitlets==5.14.3
+numba==0.61.2
+soupsieve==2.7
+cbor2==5.6.5
+multiprocess==0.70.16
+email_validator==2.2.0
+distro==1.9.0
+accelerate==1.10.0
+torchaudio==2.7.1
+jiter==0.10.0
+rfc3339-validator==0.1.4
+jupyter==1.1.1
+typer==0.16.0
+gitdb==4.0.12
+types-python-dateutil==2.9.0.20250809
+soundfile==0.13.1
+astor==0.8.1
+shellingham==1.5.4
+jupyterlab_server==2.27.3
+nvidia-curand-cu12==10.3.7.77
+jupyter-console==6.6.3
+msgpack==1.1.1
+six==1.17.0
+pycparser==2.22
+arrow==1.3.0
+tqdm==4.67.1
+lightning-utilities==0.15.2
+interegular==0.3.3
+rfc3987-syntax==1.1.0
+ipython_pygments_lexers==1.1.1
+datasets==4.0.0
+jupyter_server_terminals==0.5.3
+platformdirs==4.3.8
+pydantic-extra-types==2.10.5
+decorator==5.2.1
+mdurl==0.1.2
+nvidia-cublas-cu12==12.6.4.1
+jsonschema-specifications==2025.4.1
+setuptools==78.1.1
+fastrlock==0.8.3
+argon2-cffi-bindings==25.1.0
+nbconvert==7.16.6

BioReason_new/wandb/run-20250813_142935-iunrz6lf/files/wandb-metadata.json

@@ -0,0 +1,116 @@
+{
+  "os": "Linux-5.10.134-008.18.kangaroo.al8.x86_64-x86_64-with-glibc2.35",
+  "python": "CPython 3.11.0",
+  "startedAt": "2025-08-13T06:29:35.069039Z",
+  "args": [
+    "--text_model_name",
+    "/oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged",
+    "--protein_model_name",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m",
+    "--qformer_model_name",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft",
+    "--num_query_tokens",
+    "8",
+    "--train_dataset",
+    "/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl",
+    "--valid_dataset",
+    "/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl",
+    "--output_dir",
+    "./contrastive_outputs",
+    "--num_epochs",
+    "10",
+    "--batch_size",
+    "8",
+    "--learning_rate",
+    "1e-4",
+    "--temperature",
+    "0.07",
+    "--freeze_protein_model",
+    "--freeze_text_model",
+    "--enable_ptm",
+    "--max_length_protein",
+    "1024",
+    "--max_length_text",
+    "512",
+    "--num_workers",
+    "8",
+    "--eval_dataset",
+    "--use_wandb",
+    "--wandb_project",
+    "protein-llm-contrastive",
+    "--logging_steps",
+    "100",
+    "--eval_steps",
+    "500",
+    "--save_steps",
+    "1000"
+  ],
+  "program": "/nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py",
+  "codePath": "wangyujia/BioReason_new/train_contrastive.py",
+  "codePathLocal": "train_contrastive.py",
+  "git": {
+    "remote": "https://github.com/PorUna-byte/PAR.git",
+    "commit": "b8caf406aa1699c788f0ca6e44a1769452c317db"
+  },
+  "email": "gia0603yucca@gmail.com",
+  "root": "/nas/shared/kilab/wangyujia/BioReason_new",
+  "host": "dsw-265304-585cc9d768-ckfd7",
+  "executable": "/root/miniconda3/envs/bioreason/bin/python",
+  "cpu_count": 64,
+  "cpu_count_logical": 64,
+  "gpu": "NVIDIA A800-SXM4-80GB",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1623302262784",
+      "used": "1015083008"
+    }
+  },
+  "memory": {
+    "total": "549755813888"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-62fcc532-880f-7324-eb13-3b739a38c29b"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-af1197aa-3e7d-a53e-185d-04b1d64164da"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-3e6e551f-2d7d-a27b-654a-da56933d2283"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-a947b64a-c545-ee5e-30e6-b4b0e2dd8fdc"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-ebd04f3f-c437-1b8b-26f2-4b1ebfee93c4"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-53994891-72a6-25be-a4c8-e32c695155b0"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-a927d00a-15c5-31f5-9ae1-9d2e773821a1"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-93d555c4-29a0-cd78-abac-d98cd160d4e1"
+    }
+  ],
+  "cudaVersion": "12.1",
+  "writerId": "4obixt2d5ipbxfjgs2k7zb94y9cqikyp"
+}

BioReason_new/wandb/run-20250813_142935-iunrz6lf/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":3},"_runtime":3}

BioReason_new/wandb/run-20250813_142935-iunrz6lf/logs/debug-internal.log

@@ -0,0 +1,31 @@
+{"time":"2025-08-13T14:29:35.356008701+08:00","level":"INFO","msg":"stream: starting","core version":"0.21.1"}
+{"time":"2025-08-13T14:29:38.198330178+08:00","level":"INFO","msg":"stream: created new stream","id":"iunrz6lf"}
+{"time":"2025-08-13T14:29:38.198386468+08:00","level":"INFO","msg":"stream: started","id":"iunrz6lf"}
+{"time":"2025-08-13T14:29:38.198435503+08:00","level":"INFO","msg":"handler: started","stream_id":"iunrz6lf"}
+{"time":"2025-08-13T14:29:38.198417623+08:00","level":"INFO","msg":"writer: started","stream_id":"iunrz6lf"}
+{"time":"2025-08-13T14:29:38.19846714+08:00","level":"INFO","msg":"sender: started","stream_id":"iunrz6lf"}
+{"time":"2025-08-13T14:29:43.486140148+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062942Z&X-Goog-Expires=86399&X-Goog-Signature=2f0d6f5f9c5c89e54bc0d24076ee3ada852cc3d8927e4f1458c329da822f5b1ef992e61fb14afbb502784e633001aee460c1e6b41f9c08d0e95ed7c3672da7dfb8599fa9e963604890d5823db21bcac112368aede49bb340e45f7a2ae05b73c11e58fca0585abbae4ca6c6fdbb6bd5124551ff9eb94ce975e2970f4c330af812c20df1aadddbe0cdd69b98ff8c6f0ac38b5c63aea339dfe9e0d708634f6c82a7a9ec197f7ed2fa1f4d086a7249b559e2bee1d7bfa5f06c9539b6d015668b1cd65a2ae802066366a3d43e66a4ebf891ba39e014824efb217751f0b3ef33eca0abb2fd38b72e897d2d73a1f1037f5bbbe0cce63731dad67f458704310ed90883c7&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:33632->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062942Z&X-Goog-Expires=86399&X-Goog-Signature=2f0d6f5f9c5c89e54bc0d24076ee3ada852cc3d8927e4f1458c329da822f5b1ef992e61fb14afbb502784e633001aee460c1e6b41f9c08d0e95ed7c3672da7dfb8599fa9e963604890d5823db21bcac112368aede49bb340e45f7a2ae05b73c11e58fca0585abbae4ca6c6fdbb6bd5124551ff9eb94ce975e2970f4c330af812c20df1aadddbe0cdd69b98ff8c6f0ac38b5c63aea339dfe9e0d708634f6c82a7a9ec197f7ed2fa1f4d086a7249b559e2bee1d7bfa5f06c9539b6d015668b1cd65a2ae802066366a3d43e66a4ebf891ba39e014824efb217751f0b3ef33eca0abb2fd38b72e897d2d73a1f1037f5bbbe0cce63731dad67f458704310ed90883c7&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:29:45.374722442+08:00","level":"INFO","msg":"stream: closing","id":"iunrz6lf"}
+{"time":"2025-08-13T14:29:46.73264407+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062945Z&X-Goog-Expires=86399&X-Goog-Signature=a0cd43d546e7b1629fdd510e714b1e805fc6c949c3770f98462a4e2c829718818c94b8c4400e4ebaef7d7caf1f5a34e7f6e502cc0fb215e01033fc4e6f2fbcf335647fee8588a8372a304a03cd2d525bf8e70f9f4d7d14586a0d6e85b4b44c8b9fdd3e25d020d58267b4b70ec0b3e74c0c7d85dcca6c1e8f3f324f9d608e027c2093a116416b83e73a60665274a11408d4bd1069c15f141a588015f412feb03539a645be245abef074df01dfb9a3cf27621fd74a8d9f4c474e725f6f29a96a41b5ec75bf01c51e94ce383204248e186f5f72006f8993831019673bc3b27512dc8e255254db1f537d1d2ce8d128bd7f6549a0e89574613d96677b8079533bf978&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:51430->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062945Z&X-Goog-Expires=86399&X-Goog-Signature=a0cd43d546e7b1629fdd510e714b1e805fc6c949c3770f98462a4e2c829718818c94b8c4400e4ebaef7d7caf1f5a34e7f6e502cc0fb215e01033fc4e6f2fbcf335647fee8588a8372a304a03cd2d525bf8e70f9f4d7d14586a0d6e85b4b44c8b9fdd3e25d020d58267b4b70ec0b3e74c0c7d85dcca6c1e8f3f324f9d608e027c2093a116416b83e73a60665274a11408d4bd1069c15f141a588015f412feb03539a645be245abef074df01dfb9a3cf27621fd74a8d9f4c474e725f6f29a96a41b5ec75bf01c51e94ce383204248e186f5f72006f8993831019673bc3b27512dc8e255254db1f537d1d2ce8d128bd7f6549a0e89574613d96677b8079533bf978&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:29:49.431974131+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062946Z&X-Goog-Expires=86399&X-Goog-Signature=a9fb26867862d7a872be0fe3943b1c45fa69319d509095ad8711be6c25366fe74232e8c8d4206f570068fde288c489ba063fbcd62bc5b95346a8b7d4944d3bc9b7851a92d2cb31647084284af54cb078ea6ead75199b810ef4ed2c49532395be0fb8446e76e25d84440cad7825f70805891e6caf9ffab8dff220311a0d61928d3568c8be34b5223bb49f33d70f8cd9cb98e38c7198f94666243187d53e91a6d9f9c587b073362486fc9d2e0c32a7b254b874b26dd04e48fb1d5f3363547f2fbd353f23ba7430f8862bed5c51094227543427c5903d7b91f8b82fb1430be2359ab81a144263d199050eb36464deb35b28fd386bb04bd9350e7f4434ca7d1b4691&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:45610->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062946Z&X-Goog-Expires=86399&X-Goog-Signature=a9fb26867862d7a872be0fe3943b1c45fa69319d509095ad8711be6c25366fe74232e8c8d4206f570068fde288c489ba063fbcd62bc5b95346a8b7d4944d3bc9b7851a92d2cb31647084284af54cb078ea6ead75199b810ef4ed2c49532395be0fb8446e76e25d84440cad7825f70805891e6caf9ffab8dff220311a0d61928d3568c8be34b5223bb49f33d70f8cd9cb98e38c7198f94666243187d53e91a6d9f9c587b073362486fc9d2e0c32a7b254b874b26dd04e48fb1d5f3363547f2fbd353f23ba7430f8862bed5c51094227543427c5903d7b91f8b82fb1430be2359ab81a144263d199050eb36464deb35b28fd386bb04bd9350e7f4434ca7d1b4691&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:31:55.704727055+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062945Z&X-Goog-Expires=86399&X-Goog-Signature=08133535cc069bd54e37428d9c3e8fa600c79f990b330e9d93f9fb8eb2743163bc21ecf8894027e63c0ccefb0b8670ea098df11d3a05317c0ffdce5aa902b1bd0af7594dcebb433956dd83300e89a0936c7bab3374dc48bbb6acd43d7979e6ba0d64c714b87f20dc93305234af3bf738ec194b037406485c03f820fae18c2afde4afe64d084fd9fa9031f8a1a0b035d3857e792a0b86b87876d672348967876f1c64aa1e7399ac602d06041399a232264770f2773e3f35ce1e8a09d5cdd2c551be22c13facacd1e803de3a360bc4d7f03b43fc9d3349d7e917786769b3e0d16f959ea752b0f7c24ef872d36447e7aa1de1d09d9f8165f51eb7dd8ae557c3f7c7&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:47612->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062945Z&X-Goog-Expires=86399&X-Goog-Signature=08133535cc069bd54e37428d9c3e8fa600c79f990b330e9d93f9fb8eb2743163bc21ecf8894027e63c0ccefb0b8670ea098df11d3a05317c0ffdce5aa902b1bd0af7594dcebb433956dd83300e89a0936c7bab3374dc48bbb6acd43d7979e6ba0d64c714b87f20dc93305234af3bf738ec194b037406485c03f820fae18c2afde4afe64d084fd9fa9031f8a1a0b035d3857e792a0b86b87876d672348967876f1c64aa1e7399ac602d06041399a232264770f2773e3f35ce1e8a09d5cdd2c551be22c13facacd1e803de3a360bc4d7f03b43fc9d3349d7e917786769b3e0d16f959ea752b0f7c24ef872d36447e7aa1de1d09d9f8165f51eb7dd8ae557c3f7c7&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:31:58.201731984+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062945Z&X-Goog-Expires=86399&X-Goog-Signature=08133535cc069bd54e37428d9c3e8fa600c79f990b330e9d93f9fb8eb2743163bc21ecf8894027e63c0ccefb0b8670ea098df11d3a05317c0ffdce5aa902b1bd0af7594dcebb433956dd83300e89a0936c7bab3374dc48bbb6acd43d7979e6ba0d64c714b87f20dc93305234af3bf738ec194b037406485c03f820fae18c2afde4afe64d084fd9fa9031f8a1a0b035d3857e792a0b86b87876d672348967876f1c64aa1e7399ac602d06041399a232264770f2773e3f35ce1e8a09d5cdd2c551be22c13facacd1e803de3a360bc4d7f03b43fc9d3349d7e917786769b3e0d16f959ea752b0f7c24ef872d36447e7aa1de1d09d9f8165f51eb7dd8ae557c3f7c7&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:47248->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062945Z&X-Goog-Expires=86399&X-Goog-Signature=08133535cc069bd54e37428d9c3e8fa600c79f990b330e9d93f9fb8eb2743163bc21ecf8894027e63c0ccefb0b8670ea098df11d3a05317c0ffdce5aa902b1bd0af7594dcebb433956dd83300e89a0936c7bab3374dc48bbb6acd43d7979e6ba0d64c714b87f20dc93305234af3bf738ec194b037406485c03f820fae18c2afde4afe64d084fd9fa9031f8a1a0b035d3857e792a0b86b87876d672348967876f1c64aa1e7399ac602d06041399a232264770f2773e3f35ce1e8a09d5cdd2c551be22c13facacd1e803de3a360bc4d7f03b43fc9d3349d7e917786769b3e0d16f959ea752b0f7c24ef872d36447e7aa1de1d09d9f8165f51eb7dd8ae557c3f7c7&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:34:06.797902395+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062942Z&X-Goog-Expires=86399&X-Goog-Signature=2f0d6f5f9c5c89e54bc0d24076ee3ada852cc3d8927e4f1458c329da822f5b1ef992e61fb14afbb502784e633001aee460c1e6b41f9c08d0e95ed7c3672da7dfb8599fa9e963604890d5823db21bcac112368aede49bb340e45f7a2ae05b73c11e58fca0585abbae4ca6c6fdbb6bd5124551ff9eb94ce975e2970f4c330af812c20df1aadddbe0cdd69b98ff8c6f0ac38b5c63aea339dfe9e0d708634f6c82a7a9ec197f7ed2fa1f4d086a7249b559e2bee1d7bfa5f06c9539b6d015668b1cd65a2ae802066366a3d43e66a4ebf891ba39e014824efb217751f0b3ef33eca0abb2fd38b72e897d2d73a1f1037f5bbbe0cce63731dad67f458704310ed90883c7&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:50710->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062942Z&X-Goog-Expires=86399&X-Goog-Signature=2f0d6f5f9c5c89e54bc0d24076ee3ada852cc3d8927e4f1458c329da822f5b1ef992e61fb14afbb502784e633001aee460c1e6b41f9c08d0e95ed7c3672da7dfb8599fa9e963604890d5823db21bcac112368aede49bb340e45f7a2ae05b73c11e58fca0585abbae4ca6c6fdbb6bd5124551ff9eb94ce975e2970f4c330af812c20df1aadddbe0cdd69b98ff8c6f0ac38b5c63aea339dfe9e0d708634f6c82a7a9ec197f7ed2fa1f4d086a7249b559e2bee1d7bfa5f06c9539b6d015668b1cd65a2ae802066366a3d43e66a4ebf891ba39e014824efb217751f0b3ef33eca0abb2fd38b72e897d2d73a1f1037f5bbbe0cce63731dad67f458704310ed90883c7&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:34:10.844728612+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062945Z&X-Goog-Expires=86399&X-Goog-Signature=a0cd43d546e7b1629fdd510e714b1e805fc6c949c3770f98462a4e2c829718818c94b8c4400e4ebaef7d7caf1f5a34e7f6e502cc0fb215e01033fc4e6f2fbcf335647fee8588a8372a304a03cd2d525bf8e70f9f4d7d14586a0d6e85b4b44c8b9fdd3e25d020d58267b4b70ec0b3e74c0c7d85dcca6c1e8f3f324f9d608e027c2093a116416b83e73a60665274a11408d4bd1069c15f141a588015f412feb03539a645be245abef074df01dfb9a3cf27621fd74a8d9f4c474e725f6f29a96a41b5ec75bf01c51e94ce383204248e186f5f72006f8993831019673bc3b27512dc8e255254db1f537d1d2ce8d128bd7f6549a0e89574613d96677b8079533bf978&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:39586->142.250.73.155:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062945Z&X-Goog-Expires=86399&X-Goog-Signature=a0cd43d546e7b1629fdd510e714b1e805fc6c949c3770f98462a4e2c829718818c94b8c4400e4ebaef7d7caf1f5a34e7f6e502cc0fb215e01033fc4e6f2fbcf335647fee8588a8372a304a03cd2d525bf8e70f9f4d7d14586a0d6e85b4b44c8b9fdd3e25d020d58267b4b70ec0b3e74c0c7d85dcca6c1e8f3f324f9d608e027c2093a116416b83e73a60665274a11408d4bd1069c15f141a588015f412feb03539a645be245abef074df01dfb9a3cf27621fd74a8d9f4c474e725f6f29a96a41b5ec75bf01c51e94ce383204248e186f5f72006f8993831019673bc3b27512dc8e255254db1f537d1d2ce8d128bd7f6549a0e89574613d96677b8079533bf978&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:34:11.231196271+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062942Z&X-Goog-Expires=86399&X-Goog-Signature=2f0d6f5f9c5c89e54bc0d24076ee3ada852cc3d8927e4f1458c329da822f5b1ef992e61fb14afbb502784e633001aee460c1e6b41f9c08d0e95ed7c3672da7dfb8599fa9e963604890d5823db21bcac112368aede49bb340e45f7a2ae05b73c11e58fca0585abbae4ca6c6fdbb6bd5124551ff9eb94ce975e2970f4c330af812c20df1aadddbe0cdd69b98ff8c6f0ac38b5c63aea339dfe9e0d708634f6c82a7a9ec197f7ed2fa1f4d086a7249b559e2bee1d7bfa5f06c9539b6d015668b1cd65a2ae802066366a3d43e66a4ebf891ba39e014824efb217751f0b3ef33eca0abb2fd38b72e897d2d73a1f1037f5bbbe0cce63731dad67f458704310ed90883c7&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:40664->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062942Z&X-Goog-Expires=86399&X-Goog-Signature=2f0d6f5f9c5c89e54bc0d24076ee3ada852cc3d8927e4f1458c329da822f5b1ef992e61fb14afbb502784e633001aee460c1e6b41f9c08d0e95ed7c3672da7dfb8599fa9e963604890d5823db21bcac112368aede49bb340e45f7a2ae05b73c11e58fca0585abbae4ca6c6fdbb6bd5124551ff9eb94ce975e2970f4c330af812c20df1aadddbe0cdd69b98ff8c6f0ac38b5c63aea339dfe9e0d708634f6c82a7a9ec197f7ed2fa1f4d086a7249b559e2bee1d7bfa5f06c9539b6d015668b1cd65a2ae802066366a3d43e66a4ebf891ba39e014824efb217751f0b3ef33eca0abb2fd38b72e897d2d73a1f1037f5bbbe0cce63731dad67f458704310ed90883c7&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:34:14.997560565+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062946Z&X-Goog-Expires=86399&X-Goog-Signature=a9fb26867862d7a872be0fe3943b1c45fa69319d509095ad8711be6c25366fe74232e8c8d4206f570068fde288c489ba063fbcd62bc5b95346a8b7d4944d3bc9b7851a92d2cb31647084284af54cb078ea6ead75199b810ef4ed2c49532395be0fb8446e76e25d84440cad7825f70805891e6caf9ffab8dff220311a0d61928d3568c8be34b5223bb49f33d70f8cd9cb98e38c7198f94666243187d53e91a6d9f9c587b073362486fc9d2e0c32a7b254b874b26dd04e48fb1d5f3363547f2fbd353f23ba7430f8862bed5c51094227543427c5903d7b91f8b82fb1430be2359ab81a144263d199050eb36464deb35b28fd386bb04bd9350e7f4434ca7d1b4691&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:49688->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062946Z&X-Goog-Expires=86399&X-Goog-Signature=a9fb26867862d7a872be0fe3943b1c45fa69319d509095ad8711be6c25366fe74232e8c8d4206f570068fde288c489ba063fbcd62bc5b95346a8b7d4944d3bc9b7851a92d2cb31647084284af54cb078ea6ead75199b810ef4ed2c49532395be0fb8446e76e25d84440cad7825f70805891e6caf9ffab8dff220311a0d61928d3568c8be34b5223bb49f33d70f8cd9cb98e38c7198f94666243187d53e91a6d9f9c587b073362486fc9d2e0c32a7b254b874b26dd04e48fb1d5f3363547f2fbd353f23ba7430f8862bed5c51094227543427c5903d7b91f8b82fb1430be2359ab81a144263d199050eb36464deb35b28fd386bb04bd9350e7f4434ca7d1b4691&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:34:15.725410473+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062945Z&X-Goog-Expires=86399&X-Goog-Signature=a0cd43d546e7b1629fdd510e714b1e805fc6c949c3770f98462a4e2c829718818c94b8c4400e4ebaef7d7caf1f5a34e7f6e502cc0fb215e01033fc4e6f2fbcf335647fee8588a8372a304a03cd2d525bf8e70f9f4d7d14586a0d6e85b4b44c8b9fdd3e25d020d58267b4b70ec0b3e74c0c7d85dcca6c1e8f3f324f9d608e027c2093a116416b83e73a60665274a11408d4bd1069c15f141a588015f412feb03539a645be245abef074df01dfb9a3cf27621fd74a8d9f4c474e725f6f29a96a41b5ec75bf01c51e94ce383204248e186f5f72006f8993831019673bc3b27512dc8e255254db1f537d1d2ce8d128bd7f6549a0e89574613d96677b8079533bf978&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:40676->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062945Z&X-Goog-Expires=86399&X-Goog-Signature=a0cd43d546e7b1629fdd510e714b1e805fc6c949c3770f98462a4e2c829718818c94b8c4400e4ebaef7d7caf1f5a34e7f6e502cc0fb215e01033fc4e6f2fbcf335647fee8588a8372a304a03cd2d525bf8e70f9f4d7d14586a0d6e85b4b44c8b9fdd3e25d020d58267b4b70ec0b3e74c0c7d85dcca6c1e8f3f324f9d608e027c2093a116416b83e73a60665274a11408d4bd1069c15f141a588015f412feb03539a645be245abef074df01dfb9a3cf27621fd74a8d9f4c474e725f6f29a96a41b5ec75bf01c51e94ce383204248e186f5f72006f8993831019673bc3b27512dc8e255254db1f537d1d2ce8d128bd7f6549a0e89574613d96677b8079533bf978&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:34:21.375377934+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062942Z&X-Goog-Expires=86399&X-Goog-Signature=2f0d6f5f9c5c89e54bc0d24076ee3ada852cc3d8927e4f1458c329da822f5b1ef992e61fb14afbb502784e633001aee460c1e6b41f9c08d0e95ed7c3672da7dfb8599fa9e963604890d5823db21bcac112368aede49bb340e45f7a2ae05b73c11e58fca0585abbae4ca6c6fdbb6bd5124551ff9eb94ce975e2970f4c330af812c20df1aadddbe0cdd69b98ff8c6f0ac38b5c63aea339dfe9e0d708634f6c82a7a9ec197f7ed2fa1f4d086a7249b559e2bee1d7bfa5f06c9539b6d015668b1cd65a2ae802066366a3d43e66a4ebf891ba39e014824efb217751f0b3ef33eca0abb2fd38b72e897d2d73a1f1037f5bbbe0cce63731dad67f458704310ed90883c7&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:34200->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062942Z&X-Goog-Expires=86399&X-Goog-Signature=2f0d6f5f9c5c89e54bc0d24076ee3ada852cc3d8927e4f1458c329da822f5b1ef992e61fb14afbb502784e633001aee460c1e6b41f9c08d0e95ed7c3672da7dfb8599fa9e963604890d5823db21bcac112368aede49bb340e45f7a2ae05b73c11e58fca0585abbae4ca6c6fdbb6bd5124551ff9eb94ce975e2970f4c330af812c20df1aadddbe0cdd69b98ff8c6f0ac38b5c63aea339dfe9e0d708634f6c82a7a9ec197f7ed2fa1f4d086a7249b559e2bee1d7bfa5f06c9539b6d015668b1cd65a2ae802066366a3d43e66a4ebf891ba39e014824efb217751f0b3ef33eca0abb2fd38b72e897d2d73a1f1037f5bbbe0cce63731dad67f458704310ed90883c7&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:34:25.610475761+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062945Z&X-Goog-Expires=86399&X-Goog-Signature=a0cd43d546e7b1629fdd510e714b1e805fc6c949c3770f98462a4e2c829718818c94b8c4400e4ebaef7d7caf1f5a34e7f6e502cc0fb215e01033fc4e6f2fbcf335647fee8588a8372a304a03cd2d525bf8e70f9f4d7d14586a0d6e85b4b44c8b9fdd3e25d020d58267b4b70ec0b3e74c0c7d85dcca6c1e8f3f324f9d608e027c2093a116416b83e73a60665274a11408d4bd1069c15f141a588015f412feb03539a645be245abef074df01dfb9a3cf27621fd74a8d9f4c474e725f6f29a96a41b5ec75bf01c51e94ce383204248e186f5f72006f8993831019673bc3b27512dc8e255254db1f537d1d2ce8d128bd7f6549a0e89574613d96677b8079533bf978&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:34202->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062945Z&X-Goog-Expires=86399&X-Goog-Signature=a0cd43d546e7b1629fdd510e714b1e805fc6c949c3770f98462a4e2c829718818c94b8c4400e4ebaef7d7caf1f5a34e7f6e502cc0fb215e01033fc4e6f2fbcf335647fee8588a8372a304a03cd2d525bf8e70f9f4d7d14586a0d6e85b4b44c8b9fdd3e25d020d58267b4b70ec0b3e74c0c7d85dcca6c1e8f3f324f9d608e027c2093a116416b83e73a60665274a11408d4bd1069c15f141a588015f412feb03539a645be245abef074df01dfb9a3cf27621fd74a8d9f4c474e725f6f29a96a41b5ec75bf01c51e94ce383204248e186f5f72006f8993831019673bc3b27512dc8e255254db1f537d1d2ce8d128bd7f6549a0e89574613d96677b8079533bf978&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:34:44.404398165+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062945Z&X-Goog-Expires=86399&X-Goog-Signature=a0cd43d546e7b1629fdd510e714b1e805fc6c949c3770f98462a4e2c829718818c94b8c4400e4ebaef7d7caf1f5a34e7f6e502cc0fb215e01033fc4e6f2fbcf335647fee8588a8372a304a03cd2d525bf8e70f9f4d7d14586a0d6e85b4b44c8b9fdd3e25d020d58267b4b70ec0b3e74c0c7d85dcca6c1e8f3f324f9d608e027c2093a116416b83e73a60665274a11408d4bd1069c15f141a588015f412feb03539a645be245abef074df01dfb9a3cf27621fd74a8d9f4c474e725f6f29a96a41b5ec75bf01c51e94ce383204248e186f5f72006f8993831019673bc3b27512dc8e255254db1f537d1d2ce8d128bd7f6549a0e89574613d96677b8079533bf978&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:45466->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062945Z&X-Goog-Expires=86399&X-Goog-Signature=a0cd43d546e7b1629fdd510e714b1e805fc6c949c3770f98462a4e2c829718818c94b8c4400e4ebaef7d7caf1f5a34e7f6e502cc0fb215e01033fc4e6f2fbcf335647fee8588a8372a304a03cd2d525bf8e70f9f4d7d14586a0d6e85b4b44c8b9fdd3e25d020d58267b4b70ec0b3e74c0c7d85dcca6c1e8f3f324f9d608e027c2093a116416b83e73a60665274a11408d4bd1069c15f141a588015f412feb03539a645be245abef074df01dfb9a3cf27621fd74a8d9f4c474e725f6f29a96a41b5ec75bf01c51e94ce383204248e186f5f72006f8993831019673bc3b27512dc8e255254db1f537d1d2ce8d128bd7f6549a0e89574613d96677b8079533bf978&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:36:17.803692253+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062942Z&X-Goog-Expires=86399&X-Goog-Signature=0e1836035673f0fd54fb10b2125f785db2fa1622625c3cffb63136b19a3ad03b454542d5dd49df7faf9cd79b6e64bd767d87c72f8e4014a357770b9ad927ce9c9c26b6cd1e950e9c06265285e664b8e2d3e977f5d53377a808e46e0d979c441492a682f62274ccc9dbb452c5f131f0e19ea29a0c0b65dc6b3397e984e02c0064bce8aa2b1b53ac955e0eeb3d46b19538f4d3cf434d9e67b7f41f222f4299131c63411d9fec4b89966083e067dd522453cfc6ea3ca1fa842fceaf4fec57515c090b328e1c88d85237be6e8412cd41ff83017e917b1d00f5412c9990b8183edfd3ceaaefc4b58599935d6163ad604b023e56f100d505ccca012db8c7149b306910&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:39866->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062942Z&X-Goog-Expires=86399&X-Goog-Signature=0e1836035673f0fd54fb10b2125f785db2fa1622625c3cffb63136b19a3ad03b454542d5dd49df7faf9cd79b6e64bd767d87c72f8e4014a357770b9ad927ce9c9c26b6cd1e950e9c06265285e664b8e2d3e977f5d53377a808e46e0d979c441492a682f62274ccc9dbb452c5f131f0e19ea29a0c0b65dc6b3397e984e02c0064bce8aa2b1b53ac955e0eeb3d46b19538f4d3cf434d9e67b7f41f222f4299131c63411d9fec4b89966083e067dd522453cfc6ea3ca1fa842fceaf4fec57515c090b328e1c88d85237be6e8412cd41ff83017e917b1d00f5412c9990b8183edfd3ceaaefc4b58599935d6163ad604b023e56f100d505ccca012db8c7149b306910&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:36:20.104009313+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062942Z&X-Goog-Expires=86399&X-Goog-Signature=0e1836035673f0fd54fb10b2125f785db2fa1622625c3cffb63136b19a3ad03b454542d5dd49df7faf9cd79b6e64bd767d87c72f8e4014a357770b9ad927ce9c9c26b6cd1e950e9c06265285e664b8e2d3e977f5d53377a808e46e0d979c441492a682f62274ccc9dbb452c5f131f0e19ea29a0c0b65dc6b3397e984e02c0064bce8aa2b1b53ac955e0eeb3d46b19538f4d3cf434d9e67b7f41f222f4299131c63411d9fec4b89966083e067dd522453cfc6ea3ca1fa842fceaf4fec57515c090b328e1c88d85237be6e8412cd41ff83017e917b1d00f5412c9990b8183edfd3ceaaefc4b58599935d6163ad604b023e56f100d505ccca012db8c7149b306910&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:47998->142.250.73.155:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062942Z&X-Goog-Expires=86399&X-Goog-Signature=0e1836035673f0fd54fb10b2125f785db2fa1622625c3cffb63136b19a3ad03b454542d5dd49df7faf9cd79b6e64bd767d87c72f8e4014a357770b9ad927ce9c9c26b6cd1e950e9c06265285e664b8e2d3e977f5d53377a808e46e0d979c441492a682f62274ccc9dbb452c5f131f0e19ea29a0c0b65dc6b3397e984e02c0064bce8aa2b1b53ac955e0eeb3d46b19538f4d3cf434d9e67b7f41f222f4299131c63411d9fec4b89966083e067dd522453cfc6ea3ca1fa842fceaf4fec57515c090b328e1c88d85237be6e8412cd41ff83017e917b1d00f5412c9990b8183edfd3ceaaefc4b58599935d6163ad604b023e56f100d505ccca012db8c7149b306910&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:36:24.787732621+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062942Z&X-Goog-Expires=86399&X-Goog-Signature=0e1836035673f0fd54fb10b2125f785db2fa1622625c3cffb63136b19a3ad03b454542d5dd49df7faf9cd79b6e64bd767d87c72f8e4014a357770b9ad927ce9c9c26b6cd1e950e9c06265285e664b8e2d3e977f5d53377a808e46e0d979c441492a682f62274ccc9dbb452c5f131f0e19ea29a0c0b65dc6b3397e984e02c0064bce8aa2b1b53ac955e0eeb3d46b19538f4d3cf434d9e67b7f41f222f4299131c63411d9fec4b89966083e067dd522453cfc6ea3ca1fa842fceaf4fec57515c090b328e1c88d85237be6e8412cd41ff83017e917b1d00f5412c9990b8183edfd3ceaaefc4b58599935d6163ad604b023e56f100d505ccca012db8c7149b306910&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:59700->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062942Z&X-Goog-Expires=86399&X-Goog-Signature=0e1836035673f0fd54fb10b2125f785db2fa1622625c3cffb63136b19a3ad03b454542d5dd49df7faf9cd79b6e64bd767d87c72f8e4014a357770b9ad927ce9c9c26b6cd1e950e9c06265285e664b8e2d3e977f5d53377a808e46e0d979c441492a682f62274ccc9dbb452c5f131f0e19ea29a0c0b65dc6b3397e984e02c0064bce8aa2b1b53ac955e0eeb3d46b19538f4d3cf434d9e67b7f41f222f4299131c63411d9fec4b89966083e067dd522453cfc6ea3ca1fa842fceaf4fec57515c090b328e1c88d85237be6e8412cd41ff83017e917b1d00f5412c9990b8183edfd3ceaaefc4b58599935d6163ad604b023e56f100d505ccca012db8c7149b306910&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:36:26.015427949+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062945Z&X-Goog-Expires=86399&X-Goog-Signature=08133535cc069bd54e37428d9c3e8fa600c79f990b330e9d93f9fb8eb2743163bc21ecf8894027e63c0ccefb0b8670ea098df11d3a05317c0ffdce5aa902b1bd0af7594dcebb433956dd83300e89a0936c7bab3374dc48bbb6acd43d7979e6ba0d64c714b87f20dc93305234af3bf738ec194b037406485c03f820fae18c2afde4afe64d084fd9fa9031f8a1a0b035d3857e792a0b86b87876d672348967876f1c64aa1e7399ac602d06041399a232264770f2773e3f35ce1e8a09d5cdd2c551be22c13facacd1e803de3a360bc4d7f03b43fc9d3349d7e917786769b3e0d16f959ea752b0f7c24ef872d36447e7aa1de1d09d9f8165f51eb7dd8ae557c3f7c7&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:59716->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062945Z&X-Goog-Expires=86399&X-Goog-Signature=08133535cc069bd54e37428d9c3e8fa600c79f990b330e9d93f9fb8eb2743163bc21ecf8894027e63c0ccefb0b8670ea098df11d3a05317c0ffdce5aa902b1bd0af7594dcebb433956dd83300e89a0936c7bab3374dc48bbb6acd43d7979e6ba0d64c714b87f20dc93305234af3bf738ec194b037406485c03f820fae18c2afde4afe64d084fd9fa9031f8a1a0b035d3857e792a0b86b87876d672348967876f1c64aa1e7399ac602d06041399a232264770f2773e3f35ce1e8a09d5cdd2c551be22c13facacd1e803de3a360bc4d7f03b43fc9d3349d7e917786769b3e0d16f959ea752b0f7c24ef872d36447e7aa1de1d09d9f8165f51eb7dd8ae557c3f7c7&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:38:41.235384044+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062946Z&X-Goog-Expires=86399&X-Goog-Signature=a9fb26867862d7a872be0fe3943b1c45fa69319d509095ad8711be6c25366fe74232e8c8d4206f570068fde288c489ba063fbcd62bc5b95346a8b7d4944d3bc9b7851a92d2cb31647084284af54cb078ea6ead75199b810ef4ed2c49532395be0fb8446e76e25d84440cad7825f70805891e6caf9ffab8dff220311a0d61928d3568c8be34b5223bb49f33d70f8cd9cb98e38c7198f94666243187d53e91a6d9f9c587b073362486fc9d2e0c32a7b254b874b26dd04e48fb1d5f3363547f2fbd353f23ba7430f8862bed5c51094227543427c5903d7b91f8b82fb1430be2359ab81a144263d199050eb36464deb35b28fd386bb04bd9350e7f4434ca7d1b4691&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:58594->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062946Z&X-Goog-Expires=86399&X-Goog-Signature=a9fb26867862d7a872be0fe3943b1c45fa69319d509095ad8711be6c25366fe74232e8c8d4206f570068fde288c489ba063fbcd62bc5b95346a8b7d4944d3bc9b7851a92d2cb31647084284af54cb078ea6ead75199b810ef4ed2c49532395be0fb8446e76e25d84440cad7825f70805891e6caf9ffab8dff220311a0d61928d3568c8be34b5223bb49f33d70f8cd9cb98e38c7198f94666243187d53e91a6d9f9c587b073362486fc9d2e0c32a7b254b874b26dd04e48fb1d5f3363547f2fbd353f23ba7430f8862bed5c51094227543427c5903d7b91f8b82fb1430be2359ab81a144263d199050eb36464deb35b28fd386bb04bd9350e7f4434ca7d1b4691&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:38:45.314577369+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062945Z&X-Goog-Expires=86399&X-Goog-Signature=08133535cc069bd54e37428d9c3e8fa600c79f990b330e9d93f9fb8eb2743163bc21ecf8894027e63c0ccefb0b8670ea098df11d3a05317c0ffdce5aa902b1bd0af7594dcebb433956dd83300e89a0936c7bab3374dc48bbb6acd43d7979e6ba0d64c714b87f20dc93305234af3bf738ec194b037406485c03f820fae18c2afde4afe64d084fd9fa9031f8a1a0b035d3857e792a0b86b87876d672348967876f1c64aa1e7399ac602d06041399a232264770f2773e3f35ce1e8a09d5cdd2c551be22c13facacd1e803de3a360bc4d7f03b43fc9d3349d7e917786769b3e0d16f959ea752b0f7c24ef872d36447e7aa1de1d09d9f8165f51eb7dd8ae557c3f7c7&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:58600->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062945Z&X-Goog-Expires=86399&X-Goog-Signature=08133535cc069bd54e37428d9c3e8fa600c79f990b330e9d93f9fb8eb2743163bc21ecf8894027e63c0ccefb0b8670ea098df11d3a05317c0ffdce5aa902b1bd0af7594dcebb433956dd83300e89a0936c7bab3374dc48bbb6acd43d7979e6ba0d64c714b87f20dc93305234af3bf738ec194b037406485c03f820fae18c2afde4afe64d084fd9fa9031f8a1a0b035d3857e792a0b86b87876d672348967876f1c64aa1e7399ac602d06041399a232264770f2773e3f35ce1e8a09d5cdd2c551be22c13facacd1e803de3a360bc4d7f03b43fc9d3349d7e917786769b3e0d16f959ea752b0f7c24ef872d36447e7aa1de1d09d9f8165f51eb7dd8ae557c3f7c7&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:38:50.634715669+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062946Z&X-Goog-Expires=86399&X-Goog-Signature=a9fb26867862d7a872be0fe3943b1c45fa69319d509095ad8711be6c25366fe74232e8c8d4206f570068fde288c489ba063fbcd62bc5b95346a8b7d4944d3bc9b7851a92d2cb31647084284af54cb078ea6ead75199b810ef4ed2c49532395be0fb8446e76e25d84440cad7825f70805891e6caf9ffab8dff220311a0d61928d3568c8be34b5223bb49f33d70f8cd9cb98e38c7198f94666243187d53e91a6d9f9c587b073362486fc9d2e0c32a7b254b874b26dd04e48fb1d5f3363547f2fbd353f23ba7430f8862bed5c51094227543427c5903d7b91f8b82fb1430be2359ab81a144263d199050eb36464deb35b28fd386bb04bd9350e7f4434ca7d1b4691&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:43628->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062946Z&X-Goog-Expires=86399&X-Goog-Signature=a9fb26867862d7a872be0fe3943b1c45fa69319d509095ad8711be6c25366fe74232e8c8d4206f570068fde288c489ba063fbcd62bc5b95346a8b7d4944d3bc9b7851a92d2cb31647084284af54cb078ea6ead75199b810ef4ed2c49532395be0fb8446e76e25d84440cad7825f70805891e6caf9ffab8dff220311a0d61928d3568c8be34b5223bb49f33d70f8cd9cb98e38c7198f94666243187d53e91a6d9f9c587b073362486fc9d2e0c32a7b254b874b26dd04e48fb1d5f3363547f2fbd353f23ba7430f8862bed5c51094227543427c5903d7b91f8b82fb1430be2359ab81a144263d199050eb36464deb35b28fd386bb04bd9350e7f4434ca7d1b4691&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:39:38.564158425+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062945Z&X-Goog-Expires=86399&X-Goog-Signature=a0cd43d546e7b1629fdd510e714b1e805fc6c949c3770f98462a4e2c829718818c94b8c4400e4ebaef7d7caf1f5a34e7f6e502cc0fb215e01033fc4e6f2fbcf335647fee8588a8372a304a03cd2d525bf8e70f9f4d7d14586a0d6e85b4b44c8b9fdd3e25d020d58267b4b70ec0b3e74c0c7d85dcca6c1e8f3f324f9d608e027c2093a116416b83e73a60665274a11408d4bd1069c15f141a588015f412feb03539a645be245abef074df01dfb9a3cf27621fd74a8d9f4c474e725f6f29a96a41b5ec75bf01c51e94ce383204248e186f5f72006f8993831019673bc3b27512dc8e255254db1f537d1d2ce8d128bd7f6549a0e89574613d96677b8079533bf978&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:34124->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062945Z&X-Goog-Expires=86399&X-Goog-Signature=a0cd43d546e7b1629fdd510e714b1e805fc6c949c3770f98462a4e2c829718818c94b8c4400e4ebaef7d7caf1f5a34e7f6e502cc0fb215e01033fc4e6f2fbcf335647fee8588a8372a304a03cd2d525bf8e70f9f4d7d14586a0d6e85b4b44c8b9fdd3e25d020d58267b4b70ec0b3e74c0c7d85dcca6c1e8f3f324f9d608e027c2093a116416b83e73a60665274a11408d4bd1069c15f141a588015f412feb03539a645be245abef074df01dfb9a3cf27621fd74a8d9f4c474e725f6f29a96a41b5ec75bf01c51e94ce383204248e186f5f72006f8993831019673bc3b27512dc8e255254db1f537d1d2ce8d128bd7f6549a0e89574613d96677b8079533bf978&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:40:56.323112266+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062942Z&X-Goog-Expires=86399&X-Goog-Signature=0e1836035673f0fd54fb10b2125f785db2fa1622625c3cffb63136b19a3ad03b454542d5dd49df7faf9cd79b6e64bd767d87c72f8e4014a357770b9ad927ce9c9c26b6cd1e950e9c06265285e664b8e2d3e977f5d53377a808e46e0d979c441492a682f62274ccc9dbb452c5f131f0e19ea29a0c0b65dc6b3397e984e02c0064bce8aa2b1b53ac955e0eeb3d46b19538f4d3cf434d9e67b7f41f222f4299131c63411d9fec4b89966083e067dd522453cfc6ea3ca1fa842fceaf4fec57515c090b328e1c88d85237be6e8412cd41ff83017e917b1d00f5412c9990b8183edfd3ceaaefc4b58599935d6163ad604b023e56f100d505ccca012db8c7149b306910&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:39600->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062942Z&X-Goog-Expires=86399&X-Goog-Signature=0e1836035673f0fd54fb10b2125f785db2fa1622625c3cffb63136b19a3ad03b454542d5dd49df7faf9cd79b6e64bd767d87c72f8e4014a357770b9ad927ce9c9c26b6cd1e950e9c06265285e664b8e2d3e977f5d53377a808e46e0d979c441492a682f62274ccc9dbb452c5f131f0e19ea29a0c0b65dc6b3397e984e02c0064bce8aa2b1b53ac955e0eeb3d46b19538f4d3cf434d9e67b7f41f222f4299131c63411d9fec4b89966083e067dd522453cfc6ea3ca1fa842fceaf4fec57515c090b328e1c88d85237be6e8412cd41ff83017e917b1d00f5412c9990b8183edfd3ceaaefc4b58599935d6163ad604b023e56f100d505ccca012db8c7149b306910&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:41:08.68396866+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062942Z&X-Goog-Expires=86399&X-Goog-Signature=2f0d6f5f9c5c89e54bc0d24076ee3ada852cc3d8927e4f1458c329da822f5b1ef992e61fb14afbb502784e633001aee460c1e6b41f9c08d0e95ed7c3672da7dfb8599fa9e963604890d5823db21bcac112368aede49bb340e45f7a2ae05b73c11e58fca0585abbae4ca6c6fdbb6bd5124551ff9eb94ce975e2970f4c330af812c20df1aadddbe0cdd69b98ff8c6f0ac38b5c63aea339dfe9e0d708634f6c82a7a9ec197f7ed2fa1f4d086a7249b559e2bee1d7bfa5f06c9539b6d015668b1cd65a2ae802066366a3d43e66a4ebf891ba39e014824efb217751f0b3ef33eca0abb2fd38b72e897d2d73a1f1037f5bbbe0cce63731dad67f458704310ed90883c7&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:60998->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062942Z&X-Goog-Expires=86399&X-Goog-Signature=2f0d6f5f9c5c89e54bc0d24076ee3ada852cc3d8927e4f1458c329da822f5b1ef992e61fb14afbb502784e633001aee460c1e6b41f9c08d0e95ed7c3672da7dfb8599fa9e963604890d5823db21bcac112368aede49bb340e45f7a2ae05b73c11e58fca0585abbae4ca6c6fdbb6bd5124551ff9eb94ce975e2970f4c330af812c20df1aadddbe0cdd69b98ff8c6f0ac38b5c63aea339dfe9e0d708634f6c82a7a9ec197f7ed2fa1f4d086a7249b559e2bee1d7bfa5f06c9539b6d015668b1cd65a2ae802066366a3d43e66a4ebf891ba39e014824efb217751f0b3ef33eca0abb2fd38b72e897d2d73a1f1037f5bbbe0cce63731dad67f458704310ed90883c7&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:41:16.866321528+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062946Z&X-Goog-Expires=86399&X-Goog-Signature=a9fb26867862d7a872be0fe3943b1c45fa69319d509095ad8711be6c25366fe74232e8c8d4206f570068fde288c489ba063fbcd62bc5b95346a8b7d4944d3bc9b7851a92d2cb31647084284af54cb078ea6ead75199b810ef4ed2c49532395be0fb8446e76e25d84440cad7825f70805891e6caf9ffab8dff220311a0d61928d3568c8be34b5223bb49f33d70f8cd9cb98e38c7198f94666243187d53e91a6d9f9c587b073362486fc9d2e0c32a7b254b874b26dd04e48fb1d5f3363547f2fbd353f23ba7430f8862bed5c51094227543427c5903d7b91f8b82fb1430be2359ab81a144263d199050eb36464deb35b28fd386bb04bd9350e7f4434ca7d1b4691&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:32776->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062946Z&X-Goog-Expires=86399&X-Goog-Signature=a9fb26867862d7a872be0fe3943b1c45fa69319d509095ad8711be6c25366fe74232e8c8d4206f570068fde288c489ba063fbcd62bc5b95346a8b7d4944d3bc9b7851a92d2cb31647084284af54cb078ea6ead75199b810ef4ed2c49532395be0fb8446e76e25d84440cad7825f70805891e6caf9ffab8dff220311a0d61928d3568c8be34b5223bb49f33d70f8cd9cb98e38c7198f94666243187d53e91a6d9f9c587b073362486fc9d2e0c32a7b254b874b26dd04e48fb1d5f3363547f2fbd353f23ba7430f8862bed5c51094227543427c5903d7b91f8b82fb1430be2359ab81a144263d199050eb36464deb35b28fd386bb04bd9350e7f4434ca7d1b4691&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}

BioReason_new/wandb/run-20250813_144434-b7u1a6df/files/config.yaml

@@ -0,0 +1,200 @@
+_wandb:
+    value:
+        cli_version: 0.21.1
+        e:
+            edi3r783ioc6ivvvz46x2uehlkkl4uku:
+                args:
+                    - --text_model_name
+                    - /oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged
+                    - --protein_model_name
+                    - /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m
+                    - --qformer_model_name
+                    - /nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft
+                    - --num_query_tokens
+                    - "8"
+                    - --train_dataset
+                    - /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl
+                    - --valid_dataset
+                    - /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl
+                    - --output_dir
+                    - ./contrastive_outputs
+                    - --num_epochs
+                    - "10"
+                    - --batch_size
+                    - "8"
+                    - --learning_rate
+                    - "1e-4"
+                    - --temperature
+                    - "0.07"
+                    - --freeze_protein_model
+                    - --freeze_text_model
+                    - --enable_ptm
+                    - --max_length_protein
+                    - "1024"
+                    - --max_length_text
+                    - "512"
+                    - --num_workers
+                    - "8"
+                    - --eval_dataset
+                    - --use_wandb
+                    - --wandb_project
+                    - protein-llm-contrastive
+                    - --logging_steps
+                    - "100"
+                    - --eval_steps
+                    - "500"
+                    - --save_steps
+                    - "1000"
+                codePath: wangyujia/BioReason_new/train_contrastive.py
+                codePathLocal: train_contrastive.py
+                cpu_count: 64
+                cpu_count_logical: 64
+                cudaVersion: "12.1"
+                disk:
+                    /:
+                        total: "1623302262784"
+                        used: "1015132160"
+                email: gia0603yucca@gmail.com
+                executable: /root/miniconda3/envs/bioreason/bin/python
+                git:
+                    commit: b8caf406aa1699c788f0ca6e44a1769452c317db
+                    remote: https://github.com/PorUna-byte/PAR.git
+                gpu: NVIDIA A800-SXM4-80GB
+                gpu_count: 8
+                gpu_nvidia:
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-62fcc532-880f-7324-eb13-3b739a38c29b
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-af1197aa-3e7d-a53e-185d-04b1d64164da
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-3e6e551f-2d7d-a27b-654a-da56933d2283
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-a947b64a-c545-ee5e-30e6-b4b0e2dd8fdc
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-ebd04f3f-c437-1b8b-26f2-4b1ebfee93c4
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-53994891-72a6-25be-a4c8-e32c695155b0
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-a927d00a-15c5-31f5-9ae1-9d2e773821a1
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-93d555c4-29a0-cd78-abac-d98cd160d4e1
+                host: dsw-265304-585cc9d768-ckfd7
+                memory:
+                    total: "549755813888"
+                os: Linux-5.10.134-008.18.kangaroo.al8.x86_64-x86_64-with-glibc2.35
+                program: /nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py
+                python: CPython 3.11.0
+                root: /nas/shared/kilab/wangyujia/BioReason_new
+                startedAt: "2025-08-13T06:44:34.902815Z"
+                writerId: edi3r783ioc6ivvvz46x2uehlkkl4uku
+        m: []
+        python_version: 3.11.0
+        t:
+            "1":
+                - 1
+                - 9
+                - 11
+                - 41
+                - 49
+                - 51
+                - 71
+                - 84
+                - 98
+                - 103
+            "2":
+                - 1
+                - 9
+                - 11
+                - 41
+                - 49
+                - 51
+                - 71
+                - 84
+                - 98
+                - 103
+            "3":
+                - 13
+                - 16
+            "4": 3.11.0
+            "5": 0.21.1
+            "6": 4.55.0
+            "12": 0.21.1
+            "13": linux-x86_64
+batch_size:
+    value: 8
+bf16:
+    value: false
+cache_dir:
+    value: /model-weights
+enable_ptm:
+    value: true
+eval_dataset:
+    value: true
+eval_steps:
+    value: 500
+fp16:
+    value: false
+freeze_protein_model:
+    value: true
+freeze_text_model:
+    value: true
+gradient_accumulation_steps:
+    value: 1
+learning_rate:
+    value: 0.0001
+logging_steps:
+    value: 100
+max_length_protein:
+    value: 1024
+max_length_text:
+    value: 512
+num_epochs:
+    value: 10
+num_query_tokens:
+    value: 8
+num_workers:
+    value: 8
+output_dir:
+    value: ./contrastive_outputs
+protein_model_name:
+    value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m
+protein_weight:
+    value: 1
+ptm_weight:
+    value: 1
+qformer_model_name:
+    value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft
+save_steps:
+    value: 1000
+save_total_limit:
+    value: 3
+seed:
+    value: 42
+temperature:
+    value: 0.07
+text_model_name:
+    value: /oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged
+text_weight:
+    value: 1
+train_dataset:
+    value: /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl
+use_wandb:
+    value: true
+valid_dataset:
+    value: /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl
+wandb_entity:
+    value: null
+wandb_project:
+    value: protein-llm-contrastive
+warmup_steps:
+    value: 1000
+weight_decay:
+    value: 0.01

BioReason_new/wandb/run-20250813_144434-b7u1a6df/files/output.log

@@ -0,0 +1,40 @@
+Loading model...
+Loading checkpoint shards: 100%|██████████████████████████████████████████████████████| 4/4 [00:01<00:00,  2.62it/s]
+Some weights of EsmModel were not initialized from the model checkpoint at /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m and are newly initialized: ['pooler.dense.bias', 'pooler.dense.weight']
+You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
+Traceback (most recent call last):
+  File "/nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py", line 552, in <module>
+    trainer = main(args)
+              ^^^^^^^^^^
+  File "/nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py", line 303, in main
+    model = ProteinLLMModel(
+            ^^^^^^^^^^^^^^^^
+  File "/nas/shared/kilab/wangyujia/BioReason_new/bioreason/models/protein_llm.py", line 216, in __init__
+    self.protein_projection = QFormerProjector(
+                              ^^^^^^^^^^^^^^^^^
+  File "/nas/shared/kilab/wangyujia/BioReason_new/bioreason/models/protein_llm.py", line 63, in __init__
+    self.qformer = BertModel.from_pretrained(qformer_model_name, config=config)
+                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/modeling_utils.py", line 316, in _wrapper
+    return func(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^
+  File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/modeling_utils.py", line 5061, in from_pretrained
+    ) = cls._load_pretrained_model(
+        ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/modeling_utils.py", line 5524, in _load_pretrained_model
+    _error_msgs, disk_offload_index, cpu_offload_index = load_shard_file(args)
+                                                         ^^^^^^^^^^^^^^^^^^^^^
+  File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/modeling_utils.py", line 974, in load_shard_file
+    disk_offload_index, cpu_offload_index = _load_state_dict_into_meta_model(
+                                            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context
+    return func(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^
+  File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/modeling_utils.py", line 879, in _load_state_dict_into_meta_model
+    _load_parameter_into_model(model, param_name, param.to(param_device))
+  File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/modeling_utils.py", line 739, in _load_parameter_into_model
+    module.load_state_dict({param_type: tensor}, strict=False, assign=True)
+  File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/torch/nn/modules/module.py", line 2593, in load_state_dict
+    raise RuntimeError(
+RuntimeError: Error(s) in loading state_dict for Embedding:
+	size mismatch for weight: copying a param with shape torch.Size([512, 768]) from checkpoint, the shape in current model is torch.Size([1024, 768]).

BioReason_new/wandb/run-20250813_144434-b7u1a6df/files/requirements.txt

@@ -0,0 +1,233 @@
+jupyterlab_pygments==0.3.0
+depyf==0.19.0
+av==15.0.0
+fastapi==0.116.1
+ray==2.48.0
+cachetools==6.1.0
+prometheus_client==0.22.1
+requests==2.32.4
+jupyter-lsp==2.2.6
+jupyter-events==0.12.0
+anyio==4.10.0
+pybase64==1.4.2
+fastjsonschema==2.21.1
+cloudpickle==3.1.1
+sentry-sdk==2.34.1
+torchmetrics==1.8.1
+wheel==0.45.1
+idna==3.10
+defusedxml==0.7.1
+starlette==0.47.2
+python-dotenv==1.1.1
+comm==0.2.3
+multidict==6.6.3
+python-json-logger==3.3.0
+jupyter_server==2.16.0
+nvidia-nvtx-cu12==12.6.77
+websockets==15.0.1
+referencing==0.36.2
+gguf==0.17.1
+lm-format-enforcer==0.10.12
+xformers==0.0.31
+ninja==1.11.1.4
+beautifulsoup4==4.13.4
+dnspython==2.7.0
+certifi==2025.8.3
+markdown-it-py==3.0.0
+stack-data==0.6.3
+nvidia-cusparselt-cu12==0.6.3
+networkx==3.5
+aiohappyeyeballs==2.6.1
+nvidia-cuda-cupti-cu12==12.6.80
+llguidance==0.7.30
+rich==14.1.0
+outlines_core==0.2.10
+nvidia-nccl-cu12==2.26.2
+pandas==2.3.1
+pydantic_core==2.33.2
+argon2-cffi==25.1.0
+babel==2.17.0
+isoduration==20.11.0
+uvloop==0.21.0
+json5==0.12.0
+widgetsnbextension==4.0.14
+aiohttp==3.12.15
+notebook_shim==0.2.4
+pandocfilters==1.5.1
+vllm==0.10.0
+blake3==1.0.5
+diskcache==5.6.3
+h11==0.16.0
+attrs==25.3.0
+nvidia-cusolver-cu12==11.7.1.2
+click==8.2.1
+cupy-cuda12x==13.5.1
+jedi==0.19.2
+py-cpuinfo==9.0.0
+overrides==7.7.0
+pytz==2025.2
+protobuf==6.31.1
+dill==0.3.8
+ipykernel==6.30.1
+frozenlist==1.7.0
+huggingface-hub==0.34.4
+rignore==0.6.4
+mistune==3.1.3
+sniffio==1.3.1
+httpx==0.28.1
+sentencepiece==0.2.0
+typing_extensions==4.14.1
+typing-inspection==0.4.1
+async-lru==2.0.5
+bioreason==0.1.0
+charset-normalizer==3.4.3
+llvmlite==0.44.0
+safetensors==0.6.2
+fqdn==1.5.1
+peft==0.17.0
+fastapi-cli==0.0.8
+torch==2.7.1
+jsonschema==4.25.0
+torchvision==0.22.1
+parso==0.8.4
+pillow==11.3.0
+pycountry==24.6.1
+scipy==1.16.1
+wandb==0.21.1
+pip==25.1
+wcwidth==0.2.13
+mistral_common==1.8.3
+nvidia-cudnn-cu12==9.5.1.17
+asttokens==3.0.0
+numpy==2.2.6
+uvicorn==0.35.0
+jupyter_client==8.6.3
+debugpy==1.8.16
+fastapi-cloud-cli==0.1.5
+compressed-tensors==0.10.2
+GitPython==3.1.45
+nvidia-ml-py==13.580.65
+tiktoken==0.11.0
+partial-json-parser==0.2.1.1.post6
+prompt_toolkit==3.0.51
+uri-template==1.3.0
+pyarrow==21.0.0
+nvidia-cuda-nvrtc-cu12==12.6.77
+propcache==0.3.2
+matplotlib-inline==0.1.7
+pure_eval==0.2.3
+nest-asyncio==1.6.0
+deepspeed==0.17.4
+einops==0.8.1
+triton==3.3.1
+trl==0.21.0
+pyzmq==27.0.1
+httptools==0.6.4
+ipython==9.4.0
+packaging==25.0
+hjson==3.1.0
+soxr==0.5.0.post1
+webcolors==24.11.1
+PyYAML==6.0.2
+ipywidgets==8.1.7
+lark==1.2.2
+python-multipart==0.0.20
+tinycss2==1.4.0
+mpmath==1.3.0
+psutil==7.0.0
+nbformat==5.10.4
+qwen-vl-utils==0.0.11
+notebook==7.4.5
+jupyterlab==4.4.5
+transformers==4.55.0
+urllib3==2.5.0
+terminado==0.18.1
+xgrammar==0.1.21
+xxhash==3.5.0
+tzdata==2025.2
+jupyterlab_widgets==3.0.15
+aiosignal==1.4.0
+annotated-types==0.7.0
+nvidia-nvjitlink-cu12==12.6.85
+httpcore==1.0.9
+pexpect==4.9.0
+tornado==6.5.2
+regex==2025.7.33
+filelock==3.18.0
+nvidia-cuda-runtime-cu12==12.6.77
+openai==1.90.0
+Pygments==2.19.2
+cffi==1.17.1
+nvidia-cufft-cu12==11.3.0.4
+rpds-py==0.27.0
+rich-toolkit==0.15.0
+bleach==6.2.0
+sympy==1.14.0
+python-dateutil==2.9.0.post0
+watchfiles==1.1.0
+nbclient==0.10.2
+hf-xet==1.1.7
+Send2Trash==1.8.3
+smmap==5.0.2
+tokenizers==0.21.4
+yarl==1.20.1
+MarkupSafe==3.0.2
+prometheus-fastapi-instrumentator==7.1.0
+rfc3986-validator==0.1.1
+bitsandbytes==0.46.1
+nvidia-cufile-cu12==1.11.1.6
+fsspec==2025.3.0
+executing==2.2.0
+pydantic==2.11.7
+jupyter_core==5.8.1
+msgspec==0.19.0
+nvidia-cusparse-cu12==12.5.4.2
+opencv-python-headless==4.12.0.88
+webencodings==0.5.1
+ptyprocess==0.7.0
+pytorch-lightning==2.5.2
+Jinja2==3.1.6
+websocket-client==1.8.0
+jsonpointer==3.0.0
+traitlets==5.14.3
+numba==0.61.2
+soupsieve==2.7
+cbor2==5.6.5
+multiprocess==0.70.16
+email_validator==2.2.0
+distro==1.9.0
+accelerate==1.10.0
+torchaudio==2.7.1
+jiter==0.10.0
+rfc3339-validator==0.1.4
+jupyter==1.1.1
+typer==0.16.0
+gitdb==4.0.12
+types-python-dateutil==2.9.0.20250809
+soundfile==0.13.1
+astor==0.8.1
+shellingham==1.5.4
+jupyterlab_server==2.27.3
+nvidia-curand-cu12==10.3.7.77
+jupyter-console==6.6.3
+msgpack==1.1.1
+six==1.17.0
+pycparser==2.22
+arrow==1.3.0
+tqdm==4.67.1
+lightning-utilities==0.15.2
+interegular==0.3.3
+rfc3987-syntax==1.1.0
+ipython_pygments_lexers==1.1.1
+datasets==4.0.0
+jupyter_server_terminals==0.5.3
+platformdirs==4.3.8
+pydantic-extra-types==2.10.5
+decorator==5.2.1
+mdurl==0.1.2
+nvidia-cublas-cu12==12.6.4.1
+jsonschema-specifications==2025.4.1
+setuptools==78.1.1
+fastrlock==0.8.3
+argon2-cffi-bindings==25.1.0
+nbconvert==7.16.6

BioReason_new/wandb/run-20250813_144434-b7u1a6df/files/wandb-metadata.json

@@ -0,0 +1,116 @@
+{
+  "os": "Linux-5.10.134-008.18.kangaroo.al8.x86_64-x86_64-with-glibc2.35",
+  "python": "CPython 3.11.0",
+  "startedAt": "2025-08-13T06:44:34.902815Z",
+  "args": [
+    "--text_model_name",
+    "/oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged",
+    "--protein_model_name",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m",
+    "--qformer_model_name",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft",
+    "--num_query_tokens",
+    "8",
+    "--train_dataset",
+    "/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl",
+    "--valid_dataset",
+    "/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl",
+    "--output_dir",
+    "./contrastive_outputs",
+    "--num_epochs",
+    "10",
+    "--batch_size",
+    "8",
+    "--learning_rate",
+    "1e-4",
+    "--temperature",
+    "0.07",
+    "--freeze_protein_model",
+    "--freeze_text_model",
+    "--enable_ptm",
+    "--max_length_protein",
+    "1024",
+    "--max_length_text",
+    "512",
+    "--num_workers",
+    "8",
+    "--eval_dataset",
+    "--use_wandb",
+    "--wandb_project",
+    "protein-llm-contrastive",
+    "--logging_steps",
+    "100",
+    "--eval_steps",
+    "500",
+    "--save_steps",
+    "1000"
+  ],
+  "program": "/nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py",
+  "codePath": "wangyujia/BioReason_new/train_contrastive.py",
+  "codePathLocal": "train_contrastive.py",
+  "git": {
+    "remote": "https://github.com/PorUna-byte/PAR.git",
+    "commit": "b8caf406aa1699c788f0ca6e44a1769452c317db"
+  },
+  "email": "gia0603yucca@gmail.com",
+  "root": "/nas/shared/kilab/wangyujia/BioReason_new",
+  "host": "dsw-265304-585cc9d768-ckfd7",
+  "executable": "/root/miniconda3/envs/bioreason/bin/python",
+  "cpu_count": 64,
+  "cpu_count_logical": 64,
+  "gpu": "NVIDIA A800-SXM4-80GB",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1623302262784",
+      "used": "1015132160"
+    }
+  },
+  "memory": {
+    "total": "549755813888"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-62fcc532-880f-7324-eb13-3b739a38c29b"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-af1197aa-3e7d-a53e-185d-04b1d64164da"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-3e6e551f-2d7d-a27b-654a-da56933d2283"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-a947b64a-c545-ee5e-30e6-b4b0e2dd8fdc"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-ebd04f3f-c437-1b8b-26f2-4b1ebfee93c4"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-53994891-72a6-25be-a4c8-e32c695155b0"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-a927d00a-15c5-31f5-9ae1-9d2e773821a1"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-93d555c4-29a0-cd78-abac-d98cd160d4e1"
+    }
+  ],
+  "cudaVersion": "12.1",
+  "writerId": "edi3r783ioc6ivvvz46x2uehlkkl4uku"
+}

BioReason_new/wandb/run-20250813_144434-b7u1a6df/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":3},"_runtime":3}

BioReason_new/wandb/run-20250813_144434-b7u1a6df/logs/debug-internal.log

@@ -0,0 +1,33 @@
+{"time":"2025-08-13T14:44:35.175839156+08:00","level":"INFO","msg":"stream: starting","core version":"0.21.1"}
+{"time":"2025-08-13T14:44:37.280573541+08:00","level":"INFO","msg":"stream: created new stream","id":"b7u1a6df"}
+{"time":"2025-08-13T14:44:37.280624869+08:00","level":"INFO","msg":"stream: started","id":"b7u1a6df"}
+{"time":"2025-08-13T14:44:37.28063665+08:00","level":"INFO","msg":"handler: started","stream_id":"b7u1a6df"}
+{"time":"2025-08-13T14:44:37.280650129+08:00","level":"INFO","msg":"sender: started","stream_id":"b7u1a6df"}
+{"time":"2025-08-13T14:44:37.281034381+08:00","level":"INFO","msg":"writer: started","stream_id":"b7u1a6df"}
+{"time":"2025-08-13T14:44:39.434424587+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064438Z&X-Goog-Expires=86399&X-Goog-Signature=1c4d8e69c4a4bc16a1ecff0d2fc584e4c3edf928e99bc626ec3d49efbad4153386ec6f02ecfb9a713d077b0e7533a5e727b8841e1f183aaa50bc27bdb38788135973a93bd004290294d6805b7dead160711dcde78af1548be3f499ef000c710b6088b024dc1f374aa65daae277aa91ff8acb08d522905678e0f093d5e15a6a5d353ff3b07e46494349753f29051a19a8b88b453cf21b66006a30a00a1c086288b991685b1aa294e714697f4eee27b2175e9fcfb3c0e19cf2c191a0414c7d7c4d8d3e27a54fa2b509a430977dfc12087aa7109df8e6f999dc0c21b1f79d1973e5f6e496b36fecdef5039685395b130a02c153b53cde1288da4a2618f61b8f6f03&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:46502->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064438Z&X-Goog-Expires=86399&X-Goog-Signature=1c4d8e69c4a4bc16a1ecff0d2fc584e4c3edf928e99bc626ec3d49efbad4153386ec6f02ecfb9a713d077b0e7533a5e727b8841e1f183aaa50bc27bdb38788135973a93bd004290294d6805b7dead160711dcde78af1548be3f499ef000c710b6088b024dc1f374aa65daae277aa91ff8acb08d522905678e0f093d5e15a6a5d353ff3b07e46494349753f29051a19a8b88b453cf21b66006a30a00a1c086288b991685b1aa294e714697f4eee27b2175e9fcfb3c0e19cf2c191a0414c7d7c4d8d3e27a54fa2b509a430977dfc12087aa7109df8e6f999dc0c21b1f79d1973e5f6e496b36fecdef5039685395b130a02c153b53cde1288da4a2618f61b8f6f03&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:44:40.314088388+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064438Z&X-Goog-Expires=86399&X-Goog-Signature=b784e18b1e51706c965d5e39b31f26a5ab282191d4e34eba448ded3882706352d7cd0eb8afc78f4f6585ac2292839aa84719202f49c853ee4b4d5930137d15a5ca9c3bf336b9196fa43ea17730c012e474eb78da8dd53632305b7b46e7ce9224d376e786f4d9c6361a4c65201c804642343ee4933f6a37d8f230d25e497858b2a762c750d9b71911cc81cd8d447af1d7d384833af2cf8211ac7d6dc4b05f8b4219b6779f3e89d047d1072c8c583e36cafbb7d8151f915d1f214952244834adb7cf57459c7f14af22d01437ce43c677279de3bb3f7d40ee64b49823294ba1f30530d885cda6864edfc4e27cd2ccb101e3ca1e44f43de80cb7377595deea1aec72&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:46512->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064438Z&X-Goog-Expires=86399&X-Goog-Signature=b784e18b1e51706c965d5e39b31f26a5ab282191d4e34eba448ded3882706352d7cd0eb8afc78f4f6585ac2292839aa84719202f49c853ee4b4d5930137d15a5ca9c3bf336b9196fa43ea17730c012e474eb78da8dd53632305b7b46e7ce9224d376e786f4d9c6361a4c65201c804642343ee4933f6a37d8f230d25e497858b2a762c750d9b71911cc81cd8d447af1d7d384833af2cf8211ac7d6dc4b05f8b4219b6779f3e89d047d1072c8c583e36cafbb7d8151f915d1f214952244834adb7cf57459c7f14af22d01437ce43c677279de3bb3f7d40ee64b49823294ba1f30530d885cda6864edfc4e27cd2ccb101e3ca1e44f43de80cb7377595deea1aec72&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:44:41.741016978+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064438Z&X-Goog-Expires=86399&X-Goog-Signature=1c4d8e69c4a4bc16a1ecff0d2fc584e4c3edf928e99bc626ec3d49efbad4153386ec6f02ecfb9a713d077b0e7533a5e727b8841e1f183aaa50bc27bdb38788135973a93bd004290294d6805b7dead160711dcde78af1548be3f499ef000c710b6088b024dc1f374aa65daae277aa91ff8acb08d522905678e0f093d5e15a6a5d353ff3b07e46494349753f29051a19a8b88b453cf21b66006a30a00a1c086288b991685b1aa294e714697f4eee27b2175e9fcfb3c0e19cf2c191a0414c7d7c4d8d3e27a54fa2b509a430977dfc12087aa7109df8e6f999dc0c21b1f79d1973e5f6e496b36fecdef5039685395b130a02c153b53cde1288da4a2618f61b8f6f03&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:45294->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064438Z&X-Goog-Expires=86399&X-Goog-Signature=1c4d8e69c4a4bc16a1ecff0d2fc584e4c3edf928e99bc626ec3d49efbad4153386ec6f02ecfb9a713d077b0e7533a5e727b8841e1f183aaa50bc27bdb38788135973a93bd004290294d6805b7dead160711dcde78af1548be3f499ef000c710b6088b024dc1f374aa65daae277aa91ff8acb08d522905678e0f093d5e15a6a5d353ff3b07e46494349753f29051a19a8b88b453cf21b66006a30a00a1c086288b991685b1aa294e714697f4eee27b2175e9fcfb3c0e19cf2c191a0414c7d7c4d8d3e27a54fa2b509a430977dfc12087aa7109df8e6f999dc0c21b1f79d1973e5f6e496b36fecdef5039685395b130a02c153b53cde1288da4a2618f61b8f6f03&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:44:41.989817652+08:00","level":"INFO","msg":"stream: closing","id":"b7u1a6df"}
+{"time":"2025-08-13T14:44:42.777303637+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064438Z&X-Goog-Expires=86399&X-Goog-Signature=b784e18b1e51706c965d5e39b31f26a5ab282191d4e34eba448ded3882706352d7cd0eb8afc78f4f6585ac2292839aa84719202f49c853ee4b4d5930137d15a5ca9c3bf336b9196fa43ea17730c012e474eb78da8dd53632305b7b46e7ce9224d376e786f4d9c6361a4c65201c804642343ee4933f6a37d8f230d25e497858b2a762c750d9b71911cc81cd8d447af1d7d384833af2cf8211ac7d6dc4b05f8b4219b6779f3e89d047d1072c8c583e36cafbb7d8151f915d1f214952244834adb7cf57459c7f14af22d01437ce43c677279de3bb3f7d40ee64b49823294ba1f30530d885cda6864edfc4e27cd2ccb101e3ca1e44f43de80cb7377595deea1aec72&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:51458->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064438Z&X-Goog-Expires=86399&X-Goog-Signature=b784e18b1e51706c965d5e39b31f26a5ab282191d4e34eba448ded3882706352d7cd0eb8afc78f4f6585ac2292839aa84719202f49c853ee4b4d5930137d15a5ca9c3bf336b9196fa43ea17730c012e474eb78da8dd53632305b7b46e7ce9224d376e786f4d9c6361a4c65201c804642343ee4933f6a37d8f230d25e497858b2a762c750d9b71911cc81cd8d447af1d7d384833af2cf8211ac7d6dc4b05f8b4219b6779f3e89d047d1072c8c583e36cafbb7d8151f915d1f214952244834adb7cf57459c7f14af22d01437ce43c677279de3bb3f7d40ee64b49823294ba1f30530d885cda6864edfc4e27cd2ccb101e3ca1e44f43de80cb7377595deea1aec72&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:44:46.722453063+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064443Z&X-Goog-Expires=86399&X-Goog-Signature=a1ed9a3895a7355448222045946c9063c3c6f485bd42fdbf0f75878011690b13c231c0d4aca1dc09a5e1858884868f833ed37fe1c60079b13bd9b6bfc8819178c67a00fab449892d0eb5d54823719610deefa2051881eeed0f85c20a794ae5fbc58c588cb260634bf12a6e9adabcb55dcaa39014e44df8f6e83cdb24b4e514a211d9981878a93b681e87792d5e3d8c92786a6ba10ab50a7731f91112fbdddef9aad498d4c0513b08d85241958614cbd19074b46becfe64edc60792f69a44d99612911006d970e3514eb0813dd8af1b603c91053990e61f1ef863736debc5dac74879bbed807d2e917ea1fef6257a1be0c86b9abda3db595ba8f0776d7f59639b&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:51466->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064443Z&X-Goog-Expires=86399&X-Goog-Signature=a1ed9a3895a7355448222045946c9063c3c6f485bd42fdbf0f75878011690b13c231c0d4aca1dc09a5e1858884868f833ed37fe1c60079b13bd9b6bfc8819178c67a00fab449892d0eb5d54823719610deefa2051881eeed0f85c20a794ae5fbc58c588cb260634bf12a6e9adabcb55dcaa39014e44df8f6e83cdb24b4e514a211d9981878a93b681e87792d5e3d8c92786a6ba10ab50a7731f91112fbdddef9aad498d4c0513b08d85241958614cbd19074b46becfe64edc60792f69a44d99612911006d970e3514eb0813dd8af1b603c91053990e61f1ef863736debc5dac74879bbed807d2e917ea1fef6257a1be0c86b9abda3db595ba8f0776d7f59639b&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:44:49.307483393+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064443Z&X-Goog-Expires=86399&X-Goog-Signature=a1ed9a3895a7355448222045946c9063c3c6f485bd42fdbf0f75878011690b13c231c0d4aca1dc09a5e1858884868f833ed37fe1c60079b13bd9b6bfc8819178c67a00fab449892d0eb5d54823719610deefa2051881eeed0f85c20a794ae5fbc58c588cb260634bf12a6e9adabcb55dcaa39014e44df8f6e83cdb24b4e514a211d9981878a93b681e87792d5e3d8c92786a6ba10ab50a7731f91112fbdddef9aad498d4c0513b08d85241958614cbd19074b46becfe64edc60792f69a44d99612911006d970e3514eb0813dd8af1b603c91053990e61f1ef863736debc5dac74879bbed807d2e917ea1fef6257a1be0c86b9abda3db595ba8f0776d7f59639b&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:43308->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064443Z&X-Goog-Expires=86399&X-Goog-Signature=a1ed9a3895a7355448222045946c9063c3c6f485bd42fdbf0f75878011690b13c231c0d4aca1dc09a5e1858884868f833ed37fe1c60079b13bd9b6bfc8819178c67a00fab449892d0eb5d54823719610deefa2051881eeed0f85c20a794ae5fbc58c588cb260634bf12a6e9adabcb55dcaa39014e44df8f6e83cdb24b4e514a211d9981878a93b681e87792d5e3d8c92786a6ba10ab50a7731f91112fbdddef9aad498d4c0513b08d85241958614cbd19074b46becfe64edc60792f69a44d99612911006d970e3514eb0813dd8af1b603c91053990e61f1ef863736debc5dac74879bbed807d2e917ea1fef6257a1be0c86b9abda3db595ba8f0776d7f59639b&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:44:53.695162885+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064443Z&X-Goog-Expires=86399&X-Goog-Signature=a1ed9a3895a7355448222045946c9063c3c6f485bd42fdbf0f75878011690b13c231c0d4aca1dc09a5e1858884868f833ed37fe1c60079b13bd9b6bfc8819178c67a00fab449892d0eb5d54823719610deefa2051881eeed0f85c20a794ae5fbc58c588cb260634bf12a6e9adabcb55dcaa39014e44df8f6e83cdb24b4e514a211d9981878a93b681e87792d5e3d8c92786a6ba10ab50a7731f91112fbdddef9aad498d4c0513b08d85241958614cbd19074b46becfe64edc60792f69a44d99612911006d970e3514eb0813dd8af1b603c91053990e61f1ef863736debc5dac74879bbed807d2e917ea1fef6257a1be0c86b9abda3db595ba8f0776d7f59639b&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:46516->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064443Z&X-Goog-Expires=86399&X-Goog-Signature=a1ed9a3895a7355448222045946c9063c3c6f485bd42fdbf0f75878011690b13c231c0d4aca1dc09a5e1858884868f833ed37fe1c60079b13bd9b6bfc8819178c67a00fab449892d0eb5d54823719610deefa2051881eeed0f85c20a794ae5fbc58c588cb260634bf12a6e9adabcb55dcaa39014e44df8f6e83cdb24b4e514a211d9981878a93b681e87792d5e3d8c92786a6ba10ab50a7731f91112fbdddef9aad498d4c0513b08d85241958614cbd19074b46becfe64edc60792f69a44d99612911006d970e3514eb0813dd8af1b603c91053990e61f1ef863736debc5dac74879bbed807d2e917ea1fef6257a1be0c86b9abda3db595ba8f0776d7f59639b&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:46:52.722200106+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064442Z&X-Goog-Expires=86399&X-Goog-Signature=552d74aaa0c7a9ee8b16a055c4a3ae605223c81742d6231450f19f617797cdf1d95d8c626b624b1ae60476efdea6a6493422a74e44cfb29852257c4d29f9af812376ffea925f7202ef9d4e6d8b45406df628af59698be20fd994c7fc8357d6382e5db66115fd4506f8d3cc341450e297ca450ab1415621427e7e038e8298cf4693042ab314dda6c11ecacb5c31b9149ee121274a41ebf872a03f5d03bd65418bca7c8793eb46f19239ca0e2b83c5e22aeedb4ab15f310d51023c98305115916c47bc12887198169563b297c4539e2921fe7fadc5c61a3c195e7cc9420a952f614cbce034fb3e0a9c025e35797c41d4d91065e34db90fa48d47fab4198a2a9945&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:40186->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064442Z&X-Goog-Expires=86399&X-Goog-Signature=552d74aaa0c7a9ee8b16a055c4a3ae605223c81742d6231450f19f617797cdf1d95d8c626b624b1ae60476efdea6a6493422a74e44cfb29852257c4d29f9af812376ffea925f7202ef9d4e6d8b45406df628af59698be20fd994c7fc8357d6382e5db66115fd4506f8d3cc341450e297ca450ab1415621427e7e038e8298cf4693042ab314dda6c11ecacb5c31b9149ee121274a41ebf872a03f5d03bd65418bca7c8793eb46f19239ca0e2b83c5e22aeedb4ab15f310d51023c98305115916c47bc12887198169563b297c4539e2921fe7fadc5c61a3c195e7cc9420a952f614cbce034fb3e0a9c025e35797c41d4d91065e34db90fa48d47fab4198a2a9945&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:46:53.739297603+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064442Z&X-Goog-Expires=86399&X-Goog-Signature=18af461e43ed41e2c3a450ebab1acbb283be2f2273ca1ff2f8566f3de8aa061a57c9a976e1e832eb09904aa60e52c339d76234f48911b7414b9ccfb9d7e65ba092532fe77282e9c592d34435b6d111b8d91f81c1283fe85304b69972c3d5258038dcc9c06748efe8517ac504ccaca2aa5d3d83a205076eebb74f2a272e8dcc12cc93743bff75472b91f9d2be2716932a5ff40617c584c06bc4f2e9c267b0a1ad8dae6823f68c45a7f7bb22e8f84eab16df5228b51c5b6daf47084b2ebca10027f9117836dbfc8f62e2d96e159fac46bb346cbbeecaf4020adc3663d6c828828fbedebde155686bb7e2e7af86fdac8373bc807cbb1b0e53f0f2fd98e3d1a33643&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:40190->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064442Z&X-Goog-Expires=86399&X-Goog-Signature=18af461e43ed41e2c3a450ebab1acbb283be2f2273ca1ff2f8566f3de8aa061a57c9a976e1e832eb09904aa60e52c339d76234f48911b7414b9ccfb9d7e65ba092532fe77282e9c592d34435b6d111b8d91f81c1283fe85304b69972c3d5258038dcc9c06748efe8517ac504ccaca2aa5d3d83a205076eebb74f2a272e8dcc12cc93743bff75472b91f9d2be2716932a5ff40617c584c06bc4f2e9c267b0a1ad8dae6823f68c45a7f7bb22e8f84eab16df5228b51c5b6daf47084b2ebca10027f9117836dbfc8f62e2d96e159fac46bb346cbbeecaf4020adc3663d6c828828fbedebde155686bb7e2e7af86fdac8373bc807cbb1b0e53f0f2fd98e3d1a33643&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:47:13.20737949+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064443Z&X-Goog-Expires=86399&X-Goog-Signature=a1ed9a3895a7355448222045946c9063c3c6f485bd42fdbf0f75878011690b13c231c0d4aca1dc09a5e1858884868f833ed37fe1c60079b13bd9b6bfc8819178c67a00fab449892d0eb5d54823719610deefa2051881eeed0f85c20a794ae5fbc58c588cb260634bf12a6e9adabcb55dcaa39014e44df8f6e83cdb24b4e514a211d9981878a93b681e87792d5e3d8c92786a6ba10ab50a7731f91112fbdddef9aad498d4c0513b08d85241958614cbd19074b46becfe64edc60792f69a44d99612911006d970e3514eb0813dd8af1b603c91053990e61f1ef863736debc5dac74879bbed807d2e917ea1fef6257a1be0c86b9abda3db595ba8f0776d7f59639b&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:53132->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064443Z&X-Goog-Expires=86399&X-Goog-Signature=a1ed9a3895a7355448222045946c9063c3c6f485bd42fdbf0f75878011690b13c231c0d4aca1dc09a5e1858884868f833ed37fe1c60079b13bd9b6bfc8819178c67a00fab449892d0eb5d54823719610deefa2051881eeed0f85c20a794ae5fbc58c588cb260634bf12a6e9adabcb55dcaa39014e44df8f6e83cdb24b4e514a211d9981878a93b681e87792d5e3d8c92786a6ba10ab50a7731f91112fbdddef9aad498d4c0513b08d85241958614cbd19074b46becfe64edc60792f69a44d99612911006d970e3514eb0813dd8af1b603c91053990e61f1ef863736debc5dac74879bbed807d2e917ea1fef6257a1be0c86b9abda3db595ba8f0776d7f59639b&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:47:32.883795685+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064443Z&X-Goog-Expires=86399&X-Goog-Signature=a1ed9a3895a7355448222045946c9063c3c6f485bd42fdbf0f75878011690b13c231c0d4aca1dc09a5e1858884868f833ed37fe1c60079b13bd9b6bfc8819178c67a00fab449892d0eb5d54823719610deefa2051881eeed0f85c20a794ae5fbc58c588cb260634bf12a6e9adabcb55dcaa39014e44df8f6e83cdb24b4e514a211d9981878a93b681e87792d5e3d8c92786a6ba10ab50a7731f91112fbdddef9aad498d4c0513b08d85241958614cbd19074b46becfe64edc60792f69a44d99612911006d970e3514eb0813dd8af1b603c91053990e61f1ef863736debc5dac74879bbed807d2e917ea1fef6257a1be0c86b9abda3db595ba8f0776d7f59639b&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:48830->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064443Z&X-Goog-Expires=86399&X-Goog-Signature=a1ed9a3895a7355448222045946c9063c3c6f485bd42fdbf0f75878011690b13c231c0d4aca1dc09a5e1858884868f833ed37fe1c60079b13bd9b6bfc8819178c67a00fab449892d0eb5d54823719610deefa2051881eeed0f85c20a794ae5fbc58c588cb260634bf12a6e9adabcb55dcaa39014e44df8f6e83cdb24b4e514a211d9981878a93b681e87792d5e3d8c92786a6ba10ab50a7731f91112fbdddef9aad498d4c0513b08d85241958614cbd19074b46becfe64edc60792f69a44d99612911006d970e3514eb0813dd8af1b603c91053990e61f1ef863736debc5dac74879bbed807d2e917ea1fef6257a1be0c86b9abda3db595ba8f0776d7f59639b&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:49:07.902766533+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064438Z&X-Goog-Expires=86399&X-Goog-Signature=b784e18b1e51706c965d5e39b31f26a5ab282191d4e34eba448ded3882706352d7cd0eb8afc78f4f6585ac2292839aa84719202f49c853ee4b4d5930137d15a5ca9c3bf336b9196fa43ea17730c012e474eb78da8dd53632305b7b46e7ce9224d376e786f4d9c6361a4c65201c804642343ee4933f6a37d8f230d25e497858b2a762c750d9b71911cc81cd8d447af1d7d384833af2cf8211ac7d6dc4b05f8b4219b6779f3e89d047d1072c8c583e36cafbb7d8151f915d1f214952244834adb7cf57459c7f14af22d01437ce43c677279de3bb3f7d40ee64b49823294ba1f30530d885cda6864edfc4e27cd2ccb101e3ca1e44f43de80cb7377595deea1aec72&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:58408->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064438Z&X-Goog-Expires=86399&X-Goog-Signature=b784e18b1e51706c965d5e39b31f26a5ab282191d4e34eba448ded3882706352d7cd0eb8afc78f4f6585ac2292839aa84719202f49c853ee4b4d5930137d15a5ca9c3bf336b9196fa43ea17730c012e474eb78da8dd53632305b7b46e7ce9224d376e786f4d9c6361a4c65201c804642343ee4933f6a37d8f230d25e497858b2a762c750d9b71911cc81cd8d447af1d7d384833af2cf8211ac7d6dc4b05f8b4219b6779f3e89d047d1072c8c583e36cafbb7d8151f915d1f214952244834adb7cf57459c7f14af22d01437ce43c677279de3bb3f7d40ee64b49823294ba1f30530d885cda6864edfc4e27cd2ccb101e3ca1e44f43de80cb7377595deea1aec72&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:49:08.184696933+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064438Z&X-Goog-Expires=86399&X-Goog-Signature=1c4d8e69c4a4bc16a1ecff0d2fc584e4c3edf928e99bc626ec3d49efbad4153386ec6f02ecfb9a713d077b0e7533a5e727b8841e1f183aaa50bc27bdb38788135973a93bd004290294d6805b7dead160711dcde78af1548be3f499ef000c710b6088b024dc1f374aa65daae277aa91ff8acb08d522905678e0f093d5e15a6a5d353ff3b07e46494349753f29051a19a8b88b453cf21b66006a30a00a1c086288b991685b1aa294e714697f4eee27b2175e9fcfb3c0e19cf2c191a0414c7d7c4d8d3e27a54fa2b509a430977dfc12087aa7109df8e6f999dc0c21b1f79d1973e5f6e496b36fecdef5039685395b130a02c153b53cde1288da4a2618f61b8f6f03&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:58434->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064438Z&X-Goog-Expires=86399&X-Goog-Signature=1c4d8e69c4a4bc16a1ecff0d2fc584e4c3edf928e99bc626ec3d49efbad4153386ec6f02ecfb9a713d077b0e7533a5e727b8841e1f183aaa50bc27bdb38788135973a93bd004290294d6805b7dead160711dcde78af1548be3f499ef000c710b6088b024dc1f374aa65daae277aa91ff8acb08d522905678e0f093d5e15a6a5d353ff3b07e46494349753f29051a19a8b88b453cf21b66006a30a00a1c086288b991685b1aa294e714697f4eee27b2175e9fcfb3c0e19cf2c191a0414c7d7c4d8d3e27a54fa2b509a430977dfc12087aa7109df8e6f999dc0c21b1f79d1973e5f6e496b36fecdef5039685395b130a02c153b53cde1288da4a2618f61b8f6f03&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:49:08.238590922+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064442Z&X-Goog-Expires=86399&X-Goog-Signature=18af461e43ed41e2c3a450ebab1acbb283be2f2273ca1ff2f8566f3de8aa061a57c9a976e1e832eb09904aa60e52c339d76234f48911b7414b9ccfb9d7e65ba092532fe77282e9c592d34435b6d111b8d91f81c1283fe85304b69972c3d5258038dcc9c06748efe8517ac504ccaca2aa5d3d83a205076eebb74f2a272e8dcc12cc93743bff75472b91f9d2be2716932a5ff40617c584c06bc4f2e9c267b0a1ad8dae6823f68c45a7f7bb22e8f84eab16df5228b51c5b6daf47084b2ebca10027f9117836dbfc8f62e2d96e159fac46bb346cbbeecaf4020adc3663d6c828828fbedebde155686bb7e2e7af86fdac8373bc807cbb1b0e53f0f2fd98e3d1a33643&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:58410->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064442Z&X-Goog-Expires=86399&X-Goog-Signature=18af461e43ed41e2c3a450ebab1acbb283be2f2273ca1ff2f8566f3de8aa061a57c9a976e1e832eb09904aa60e52c339d76234f48911b7414b9ccfb9d7e65ba092532fe77282e9c592d34435b6d111b8d91f81c1283fe85304b69972c3d5258038dcc9c06748efe8517ac504ccaca2aa5d3d83a205076eebb74f2a272e8dcc12cc93743bff75472b91f9d2be2716932a5ff40617c584c06bc4f2e9c267b0a1ad8dae6823f68c45a7f7bb22e8f84eab16df5228b51c5b6daf47084b2ebca10027f9117836dbfc8f62e2d96e159fac46bb346cbbeecaf4020adc3663d6c828828fbedebde155686bb7e2e7af86fdac8373bc807cbb1b0e53f0f2fd98e3d1a33643&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:49:13.157250807+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064442Z&X-Goog-Expires=86399&X-Goog-Signature=18af461e43ed41e2c3a450ebab1acbb283be2f2273ca1ff2f8566f3de8aa061a57c9a976e1e832eb09904aa60e52c339d76234f48911b7414b9ccfb9d7e65ba092532fe77282e9c592d34435b6d111b8d91f81c1283fe85304b69972c3d5258038dcc9c06748efe8517ac504ccaca2aa5d3d83a205076eebb74f2a272e8dcc12cc93743bff75472b91f9d2be2716932a5ff40617c584c06bc4f2e9c267b0a1ad8dae6823f68c45a7f7bb22e8f84eab16df5228b51c5b6daf47084b2ebca10027f9117836dbfc8f62e2d96e159fac46bb346cbbeecaf4020adc3663d6c828828fbedebde155686bb7e2e7af86fdac8373bc807cbb1b0e53f0f2fd98e3d1a33643&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:58436->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064442Z&X-Goog-Expires=86399&X-Goog-Signature=18af461e43ed41e2c3a450ebab1acbb283be2f2273ca1ff2f8566f3de8aa061a57c9a976e1e832eb09904aa60e52c339d76234f48911b7414b9ccfb9d7e65ba092532fe77282e9c592d34435b6d111b8d91f81c1283fe85304b69972c3d5258038dcc9c06748efe8517ac504ccaca2aa5d3d83a205076eebb74f2a272e8dcc12cc93743bff75472b91f9d2be2716932a5ff40617c584c06bc4f2e9c267b0a1ad8dae6823f68c45a7f7bb22e8f84eab16df5228b51c5b6daf47084b2ebca10027f9117836dbfc8f62e2d96e159fac46bb346cbbeecaf4020adc3663d6c828828fbedebde155686bb7e2e7af86fdac8373bc807cbb1b0e53f0f2fd98e3d1a33643&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:49:18.001715505+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064438Z&X-Goog-Expires=86399&X-Goog-Signature=1c4d8e69c4a4bc16a1ecff0d2fc584e4c3edf928e99bc626ec3d49efbad4153386ec6f02ecfb9a713d077b0e7533a5e727b8841e1f183aaa50bc27bdb38788135973a93bd004290294d6805b7dead160711dcde78af1548be3f499ef000c710b6088b024dc1f374aa65daae277aa91ff8acb08d522905678e0f093d5e15a6a5d353ff3b07e46494349753f29051a19a8b88b453cf21b66006a30a00a1c086288b991685b1aa294e714697f4eee27b2175e9fcfb3c0e19cf2c191a0414c7d7c4d8d3e27a54fa2b509a430977dfc12087aa7109df8e6f999dc0c21b1f79d1973e5f6e496b36fecdef5039685395b130a02c153b53cde1288da4a2618f61b8f6f03&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:36496->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064438Z&X-Goog-Expires=86399&X-Goog-Signature=1c4d8e69c4a4bc16a1ecff0d2fc584e4c3edf928e99bc626ec3d49efbad4153386ec6f02ecfb9a713d077b0e7533a5e727b8841e1f183aaa50bc27bdb38788135973a93bd004290294d6805b7dead160711dcde78af1548be3f499ef000c710b6088b024dc1f374aa65daae277aa91ff8acb08d522905678e0f093d5e15a6a5d353ff3b07e46494349753f29051a19a8b88b453cf21b66006a30a00a1c086288b991685b1aa294e714697f4eee27b2175e9fcfb3c0e19cf2c191a0414c7d7c4d8d3e27a54fa2b509a430977dfc12087aa7109df8e6f999dc0c21b1f79d1973e5f6e496b36fecdef5039685395b130a02c153b53cde1288da4a2618f61b8f6f03&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:51:14.898455084+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064442Z&X-Goog-Expires=86399&X-Goog-Signature=552d74aaa0c7a9ee8b16a055c4a3ae605223c81742d6231450f19f617797cdf1d95d8c626b624b1ae60476efdea6a6493422a74e44cfb29852257c4d29f9af812376ffea925f7202ef9d4e6d8b45406df628af59698be20fd994c7fc8357d6382e5db66115fd4506f8d3cc341450e297ca450ab1415621427e7e038e8298cf4693042ab314dda6c11ecacb5c31b9149ee121274a41ebf872a03f5d03bd65418bca7c8793eb46f19239ca0e2b83c5e22aeedb4ab15f310d51023c98305115916c47bc12887198169563b297c4539e2921fe7fadc5c61a3c195e7cc9420a952f614cbce034fb3e0a9c025e35797c41d4d91065e34db90fa48d47fab4198a2a9945&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:48578->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064442Z&X-Goog-Expires=86399&X-Goog-Signature=552d74aaa0c7a9ee8b16a055c4a3ae605223c81742d6231450f19f617797cdf1d95d8c626b624b1ae60476efdea6a6493422a74e44cfb29852257c4d29f9af812376ffea925f7202ef9d4e6d8b45406df628af59698be20fd994c7fc8357d6382e5db66115fd4506f8d3cc341450e297ca450ab1415621427e7e038e8298cf4693042ab314dda6c11ecacb5c31b9149ee121274a41ebf872a03f5d03bd65418bca7c8793eb46f19239ca0e2b83c5e22aeedb4ab15f310d51023c98305115916c47bc12887198169563b297c4539e2921fe7fadc5c61a3c195e7cc9420a952f614cbce034fb3e0a9c025e35797c41d4d91065e34db90fa48d47fab4198a2a9945&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:51:31.292954507+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064442Z&X-Goog-Expires=86399&X-Goog-Signature=18af461e43ed41e2c3a450ebab1acbb283be2f2273ca1ff2f8566f3de8aa061a57c9a976e1e832eb09904aa60e52c339d76234f48911b7414b9ccfb9d7e65ba092532fe77282e9c592d34435b6d111b8d91f81c1283fe85304b69972c3d5258038dcc9c06748efe8517ac504ccaca2aa5d3d83a205076eebb74f2a272e8dcc12cc93743bff75472b91f9d2be2716932a5ff40617c584c06bc4f2e9c267b0a1ad8dae6823f68c45a7f7bb22e8f84eab16df5228b51c5b6daf47084b2ebca10027f9117836dbfc8f62e2d96e159fac46bb346cbbeecaf4020adc3663d6c828828fbedebde155686bb7e2e7af86fdac8373bc807cbb1b0e53f0f2fd98e3d1a33643&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:49362->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064442Z&X-Goog-Expires=86399&X-Goog-Signature=18af461e43ed41e2c3a450ebab1acbb283be2f2273ca1ff2f8566f3de8aa061a57c9a976e1e832eb09904aa60e52c339d76234f48911b7414b9ccfb9d7e65ba092532fe77282e9c592d34435b6d111b8d91f81c1283fe85304b69972c3d5258038dcc9c06748efe8517ac504ccaca2aa5d3d83a205076eebb74f2a272e8dcc12cc93743bff75472b91f9d2be2716932a5ff40617c584c06bc4f2e9c267b0a1ad8dae6823f68c45a7f7bb22e8f84eab16df5228b51c5b6daf47084b2ebca10027f9117836dbfc8f62e2d96e159fac46bb346cbbeecaf4020adc3663d6c828828fbedebde155686bb7e2e7af86fdac8373bc807cbb1b0e53f0f2fd98e3d1a33643&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:51:48.908372115+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064442Z&X-Goog-Expires=86399&X-Goog-Signature=18af461e43ed41e2c3a450ebab1acbb283be2f2273ca1ff2f8566f3de8aa061a57c9a976e1e832eb09904aa60e52c339d76234f48911b7414b9ccfb9d7e65ba092532fe77282e9c592d34435b6d111b8d91f81c1283fe85304b69972c3d5258038dcc9c06748efe8517ac504ccaca2aa5d3d83a205076eebb74f2a272e8dcc12cc93743bff75472b91f9d2be2716932a5ff40617c584c06bc4f2e9c267b0a1ad8dae6823f68c45a7f7bb22e8f84eab16df5228b51c5b6daf47084b2ebca10027f9117836dbfc8f62e2d96e159fac46bb346cbbeecaf4020adc3663d6c828828fbedebde155686bb7e2e7af86fdac8373bc807cbb1b0e53f0f2fd98e3d1a33643&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:38778->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064442Z&X-Goog-Expires=86399&X-Goog-Signature=18af461e43ed41e2c3a450ebab1acbb283be2f2273ca1ff2f8566f3de8aa061a57c9a976e1e832eb09904aa60e52c339d76234f48911b7414b9ccfb9d7e65ba092532fe77282e9c592d34435b6d111b8d91f81c1283fe85304b69972c3d5258038dcc9c06748efe8517ac504ccaca2aa5d3d83a205076eebb74f2a272e8dcc12cc93743bff75472b91f9d2be2716932a5ff40617c584c06bc4f2e9c267b0a1ad8dae6823f68c45a7f7bb22e8f84eab16df5228b51c5b6daf47084b2ebca10027f9117836dbfc8f62e2d96e159fac46bb346cbbeecaf4020adc3663d6c828828fbedebde155686bb7e2e7af86fdac8373bc807cbb1b0e53f0f2fd98e3d1a33643&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:52:22.529468861+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064442Z&X-Goog-Expires=86399&X-Goog-Signature=18af461e43ed41e2c3a450ebab1acbb283be2f2273ca1ff2f8566f3de8aa061a57c9a976e1e832eb09904aa60e52c339d76234f48911b7414b9ccfb9d7e65ba092532fe77282e9c592d34435b6d111b8d91f81c1283fe85304b69972c3d5258038dcc9c06748efe8517ac504ccaca2aa5d3d83a205076eebb74f2a272e8dcc12cc93743bff75472b91f9d2be2716932a5ff40617c584c06bc4f2e9c267b0a1ad8dae6823f68c45a7f7bb22e8f84eab16df5228b51c5b6daf47084b2ebca10027f9117836dbfc8f62e2d96e159fac46bb346cbbeecaf4020adc3663d6c828828fbedebde155686bb7e2e7af86fdac8373bc807cbb1b0e53f0f2fd98e3d1a33643&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:49956->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064442Z&X-Goog-Expires=86399&X-Goog-Signature=18af461e43ed41e2c3a450ebab1acbb283be2f2273ca1ff2f8566f3de8aa061a57c9a976e1e832eb09904aa60e52c339d76234f48911b7414b9ccfb9d7e65ba092532fe77282e9c592d34435b6d111b8d91f81c1283fe85304b69972c3d5258038dcc9c06748efe8517ac504ccaca2aa5d3d83a205076eebb74f2a272e8dcc12cc93743bff75472b91f9d2be2716932a5ff40617c584c06bc4f2e9c267b0a1ad8dae6823f68c45a7f7bb22e8f84eab16df5228b51c5b6daf47084b2ebca10027f9117836dbfc8f62e2d96e159fac46bb346cbbeecaf4020adc3663d6c828828fbedebde155686bb7e2e7af86fdac8373bc807cbb1b0e53f0f2fd98e3d1a33643&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:52:28.74584019+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064443Z&X-Goog-Expires=86399&X-Goog-Signature=a1ed9a3895a7355448222045946c9063c3c6f485bd42fdbf0f75878011690b13c231c0d4aca1dc09a5e1858884868f833ed37fe1c60079b13bd9b6bfc8819178c67a00fab449892d0eb5d54823719610deefa2051881eeed0f85c20a794ae5fbc58c588cb260634bf12a6e9adabcb55dcaa39014e44df8f6e83cdb24b4e514a211d9981878a93b681e87792d5e3d8c92786a6ba10ab50a7731f91112fbdddef9aad498d4c0513b08d85241958614cbd19074b46becfe64edc60792f69a44d99612911006d970e3514eb0813dd8af1b603c91053990e61f1ef863736debc5dac74879bbed807d2e917ea1fef6257a1be0c86b9abda3db595ba8f0776d7f59639b&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:55480->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064443Z&X-Goog-Expires=86399&X-Goog-Signature=a1ed9a3895a7355448222045946c9063c3c6f485bd42fdbf0f75878011690b13c231c0d4aca1dc09a5e1858884868f833ed37fe1c60079b13bd9b6bfc8819178c67a00fab449892d0eb5d54823719610deefa2051881eeed0f85c20a794ae5fbc58c588cb260634bf12a6e9adabcb55dcaa39014e44df8f6e83cdb24b4e514a211d9981878a93b681e87792d5e3d8c92786a6ba10ab50a7731f91112fbdddef9aad498d4c0513b08d85241958614cbd19074b46becfe64edc60792f69a44d99612911006d970e3514eb0813dd8af1b603c91053990e61f1ef863736debc5dac74879bbed807d2e917ea1fef6257a1be0c86b9abda3db595ba8f0776d7f59639b&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:53:29.135553591+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064443Z&X-Goog-Expires=86399&X-Goog-Signature=a1ed9a3895a7355448222045946c9063c3c6f485bd42fdbf0f75878011690b13c231c0d4aca1dc09a5e1858884868f833ed37fe1c60079b13bd9b6bfc8819178c67a00fab449892d0eb5d54823719610deefa2051881eeed0f85c20a794ae5fbc58c588cb260634bf12a6e9adabcb55dcaa39014e44df8f6e83cdb24b4e514a211d9981878a93b681e87792d5e3d8c92786a6ba10ab50a7731f91112fbdddef9aad498d4c0513b08d85241958614cbd19074b46becfe64edc60792f69a44d99612911006d970e3514eb0813dd8af1b603c91053990e61f1ef863736debc5dac74879bbed807d2e917ea1fef6257a1be0c86b9abda3db595ba8f0776d7f59639b&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:48996->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064443Z&X-Goog-Expires=86399&X-Goog-Signature=a1ed9a3895a7355448222045946c9063c3c6f485bd42fdbf0f75878011690b13c231c0d4aca1dc09a5e1858884868f833ed37fe1c60079b13bd9b6bfc8819178c67a00fab449892d0eb5d54823719610deefa2051881eeed0f85c20a794ae5fbc58c588cb260634bf12a6e9adabcb55dcaa39014e44df8f6e83cdb24b4e514a211d9981878a93b681e87792d5e3d8c92786a6ba10ab50a7731f91112fbdddef9aad498d4c0513b08d85241958614cbd19074b46becfe64edc60792f69a44d99612911006d970e3514eb0813dd8af1b603c91053990e61f1ef863736debc5dac74879bbed807d2e917ea1fef6257a1be0c86b9abda3db595ba8f0776d7f59639b&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:53:38.183430588+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064438Z&X-Goog-Expires=86399&X-Goog-Signature=b784e18b1e51706c965d5e39b31f26a5ab282191d4e34eba448ded3882706352d7cd0eb8afc78f4f6585ac2292839aa84719202f49c853ee4b4d5930137d15a5ca9c3bf336b9196fa43ea17730c012e474eb78da8dd53632305b7b46e7ce9224d376e786f4d9c6361a4c65201c804642343ee4933f6a37d8f230d25e497858b2a762c750d9b71911cc81cd8d447af1d7d384833af2cf8211ac7d6dc4b05f8b4219b6779f3e89d047d1072c8c583e36cafbb7d8151f915d1f214952244834adb7cf57459c7f14af22d01437ce43c677279de3bb3f7d40ee64b49823294ba1f30530d885cda6864edfc4e27cd2ccb101e3ca1e44f43de80cb7377595deea1aec72&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:45444->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064438Z&X-Goog-Expires=86399&X-Goog-Signature=b784e18b1e51706c965d5e39b31f26a5ab282191d4e34eba448ded3882706352d7cd0eb8afc78f4f6585ac2292839aa84719202f49c853ee4b4d5930137d15a5ca9c3bf336b9196fa43ea17730c012e474eb78da8dd53632305b7b46e7ce9224d376e786f4d9c6361a4c65201c804642343ee4933f6a37d8f230d25e497858b2a762c750d9b71911cc81cd8d447af1d7d384833af2cf8211ac7d6dc4b05f8b4219b6779f3e89d047d1072c8c583e36cafbb7d8151f915d1f214952244834adb7cf57459c7f14af22d01437ce43c677279de3bb3f7d40ee64b49823294ba1f30530d885cda6864edfc4e27cd2ccb101e3ca1e44f43de80cb7377595deea1aec72&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:53:58.779409183+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064438Z&X-Goog-Expires=86399&X-Goog-Signature=1c4d8e69c4a4bc16a1ecff0d2fc584e4c3edf928e99bc626ec3d49efbad4153386ec6f02ecfb9a713d077b0e7533a5e727b8841e1f183aaa50bc27bdb38788135973a93bd004290294d6805b7dead160711dcde78af1548be3f499ef000c710b6088b024dc1f374aa65daae277aa91ff8acb08d522905678e0f093d5e15a6a5d353ff3b07e46494349753f29051a19a8b88b453cf21b66006a30a00a1c086288b991685b1aa294e714697f4eee27b2175e9fcfb3c0e19cf2c191a0414c7d7c4d8d3e27a54fa2b509a430977dfc12087aa7109df8e6f999dc0c21b1f79d1973e5f6e496b36fecdef5039685395b130a02c153b53cde1288da4a2618f61b8f6f03&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:52344->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064438Z&X-Goog-Expires=86399&X-Goog-Signature=1c4d8e69c4a4bc16a1ecff0d2fc584e4c3edf928e99bc626ec3d49efbad4153386ec6f02ecfb9a713d077b0e7533a5e727b8841e1f183aaa50bc27bdb38788135973a93bd004290294d6805b7dead160711dcde78af1548be3f499ef000c710b6088b024dc1f374aa65daae277aa91ff8acb08d522905678e0f093d5e15a6a5d353ff3b07e46494349753f29051a19a8b88b453cf21b66006a30a00a1c086288b991685b1aa294e714697f4eee27b2175e9fcfb3c0e19cf2c191a0414c7d7c4d8d3e27a54fa2b509a430977dfc12087aa7109df8e6f999dc0c21b1f79d1973e5f6e496b36fecdef5039685395b130a02c153b53cde1288da4a2618f61b8f6f03&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:55:32.951218418+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064442Z&X-Goog-Expires=86399&X-Goog-Signature=18af461e43ed41e2c3a450ebab1acbb283be2f2273ca1ff2f8566f3de8aa061a57c9a976e1e832eb09904aa60e52c339d76234f48911b7414b9ccfb9d7e65ba092532fe77282e9c592d34435b6d111b8d91f81c1283fe85304b69972c3d5258038dcc9c06748efe8517ac504ccaca2aa5d3d83a205076eebb74f2a272e8dcc12cc93743bff75472b91f9d2be2716932a5ff40617c584c06bc4f2e9c267b0a1ad8dae6823f68c45a7f7bb22e8f84eab16df5228b51c5b6daf47084b2ebca10027f9117836dbfc8f62e2d96e159fac46bb346cbbeecaf4020adc3663d6c828828fbedebde155686bb7e2e7af86fdac8373bc807cbb1b0e53f0f2fd98e3d1a33643&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:33980->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064442Z&X-Goog-Expires=86399&X-Goog-Signature=18af461e43ed41e2c3a450ebab1acbb283be2f2273ca1ff2f8566f3de8aa061a57c9a976e1e832eb09904aa60e52c339d76234f48911b7414b9ccfb9d7e65ba092532fe77282e9c592d34435b6d111b8d91f81c1283fe85304b69972c3d5258038dcc9c06748efe8517ac504ccaca2aa5d3d83a205076eebb74f2a272e8dcc12cc93743bff75472b91f9d2be2716932a5ff40617c584c06bc4f2e9c267b0a1ad8dae6823f68c45a7f7bb22e8f84eab16df5228b51c5b6daf47084b2ebca10027f9117836dbfc8f62e2d96e159fac46bb346cbbeecaf4020adc3663d6c828828fbedebde155686bb7e2e7af86fdac8373bc807cbb1b0e53f0f2fd98e3d1a33643&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:55:41.170491233+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064442Z&X-Goog-Expires=86399&X-Goog-Signature=552d74aaa0c7a9ee8b16a055c4a3ae605223c81742d6231450f19f617797cdf1d95d8c626b624b1ae60476efdea6a6493422a74e44cfb29852257c4d29f9af812376ffea925f7202ef9d4e6d8b45406df628af59698be20fd994c7fc8357d6382e5db66115fd4506f8d3cc341450e297ca450ab1415621427e7e038e8298cf4693042ab314dda6c11ecacb5c31b9149ee121274a41ebf872a03f5d03bd65418bca7c8793eb46f19239ca0e2b83c5e22aeedb4ab15f310d51023c98305115916c47bc12887198169563b297c4539e2921fe7fadc5c61a3c195e7cc9420a952f614cbce034fb3e0a9c025e35797c41d4d91065e34db90fa48d47fab4198a2a9945&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:52256->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064442Z&X-Goog-Expires=86399&X-Goog-Signature=552d74aaa0c7a9ee8b16a055c4a3ae605223c81742d6231450f19f617797cdf1d95d8c626b624b1ae60476efdea6a6493422a74e44cfb29852257c4d29f9af812376ffea925f7202ef9d4e6d8b45406df628af59698be20fd994c7fc8357d6382e5db66115fd4506f8d3cc341450e297ca450ab1415621427e7e038e8298cf4693042ab314dda6c11ecacb5c31b9149ee121274a41ebf872a03f5d03bd65418bca7c8793eb46f19239ca0e2b83c5e22aeedb4ab15f310d51023c98305115916c47bc12887198169563b297c4539e2921fe7fadc5c61a3c195e7cc9420a952f614cbce034fb3e0a9c025e35797c41d4d91065e34db90fa48d47fab4198a2a9945&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}

BioReason_new/wandb/run-20250813_145627-ep0ib6n2/files/config.yaml

@@ -0,0 +1,458 @@
+_wandb:
+    value:
+        cli_version: 0.21.1
+        e:
+            8kepep2ysih7kn77h1iftipfg3shqy5q:
+                args:
+                    - --text_model_name
+                    - /oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged
+                    - --protein_model_name
+                    - /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m
+                    - --qformer_model_name
+                    - /nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft
+                    - --num_query_tokens
+                    - "8"
+                    - --train_dataset
+                    - /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl
+                    - --valid_dataset
+                    - /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl
+                    - --output_dir
+                    - ./contrastive_outputs
+                    - --num_epochs
+                    - "10"
+                    - --batch_size
+                    - "8"
+                    - --learning_rate
+                    - "1e-4"
+                    - --temperature
+                    - "0.07"
+                    - --freeze_protein_model
+                    - --freeze_text_model
+                    - --enable_ptm
+                    - --max_length_protein
+                    - "1024"
+                    - --max_length_text
+                    - "512"
+                    - --num_workers
+                    - "8"
+                    - --eval_dataset
+                    - --use_wandb
+                    - --wandb_project
+                    - protein-llm-contrastive
+                    - --logging_steps
+                    - "100"
+                    - --eval_steps
+                    - "500"
+                    - --save_steps
+                    - "1000"
+                codePath: wangyujia/BioReason_new/train_contrastive.py
+                codePathLocal: train_contrastive.py
+                cpu_count: 64
+                cpu_count_logical: 64
+                cudaVersion: "12.1"
+                disk:
+                    /:
+                        total: "1623302262784"
+                        used: "1015189504"
+                email: gia0603yucca@gmail.com
+                executable: /root/miniconda3/envs/bioreason/bin/python
+                git:
+                    commit: b8caf406aa1699c788f0ca6e44a1769452c317db
+                    remote: https://github.com/PorUna-byte/PAR.git
+                gpu: NVIDIA A800-SXM4-80GB
+                gpu_count: 8
+                gpu_nvidia:
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-62fcc532-880f-7324-eb13-3b739a38c29b
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-af1197aa-3e7d-a53e-185d-04b1d64164da
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-3e6e551f-2d7d-a27b-654a-da56933d2283
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-a947b64a-c545-ee5e-30e6-b4b0e2dd8fdc
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-ebd04f3f-c437-1b8b-26f2-4b1ebfee93c4
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-53994891-72a6-25be-a4c8-e32c695155b0
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-a927d00a-15c5-31f5-9ae1-9d2e773821a1
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-93d555c4-29a0-cd78-abac-d98cd160d4e1
+                host: dsw-265304-585cc9d768-ckfd7
+                memory:
+                    total: "549755813888"
+                os: Linux-5.10.134-008.18.kangaroo.al8.x86_64-x86_64-with-glibc2.35
+                program: /nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py
+                python: CPython 3.11.0
+                root: /nas/shared/kilab/wangyujia/BioReason_new
+                startedAt: "2025-08-13T06:56:27.037635Z"
+                writerId: 8kepep2ysih7kn77h1iftipfg3shqy5q
+        m:
+            - "1": train/global_step
+              "6":
+                - 3
+              "7": []
+            - "2": '*'
+              "5": 1
+              "6":
+                - 1
+              "7": []
+        python_version: 3.11.0
+        t:
+            "1":
+                - 1
+                - 9
+                - 11
+                - 41
+                - 49
+                - 51
+                - 71
+                - 84
+                - 98
+                - 103
+            "2":
+                - 1
+                - 9
+                - 11
+                - 41
+                - 49
+                - 51
+                - 71
+                - 84
+                - 98
+                - 103
+            "3":
+                - 7
+                - 13
+                - 16
+                - 66
+            "4": 3.11.0
+            "5": 0.21.1
+            "6": 4.55.0
+            "9":
+                "1": transformers_trainer
+            "12": 0.21.1
+            "13": linux-x86_64
+accelerator_config:
+    value:
+        dispatch_batches: null
+        even_batches: true
+        gradient_accumulation_kwargs: null
+        non_blocking: false
+        split_batches: false
+        use_seedable_sampler: true
+adafactor:
+    value: false
+adam_beta1:
+    value: 0.9
+adam_beta2:
+    value: 0.999
+adam_epsilon:
+    value: 1e-08
+auto_find_batch_size:
+    value: false
+average_tokens_across_devices:
+    value: false
+batch_eval_metrics:
+    value: false
+batch_size:
+    value: 8
+bf16:
+    value: false
+bf16_full_eval:
+    value: false
+cache_dir:
+    value: /model-weights
+data_seed:
+    value: null
+dataloader_drop_last:
+    value: false
+dataloader_num_workers:
+    value: 8
+dataloader_persistent_workers:
+    value: false
+dataloader_pin_memory:
+    value: true
+dataloader_prefetch_factor:
+    value: null
+ddp_backend:
+    value: null
+ddp_broadcast_buffers:
+    value: null
+ddp_bucket_cap_mb:
+    value: null
+ddp_find_unused_parameters:
+    value: false
+ddp_timeout:
+    value: 1800
+debug:
+    value: []
+deepspeed:
+    value: null
+disable_tqdm:
+    value: false
+do_eval:
+    value: true
+do_predict:
+    value: false
+do_train:
+    value: false
+enable_ptm:
+    value: true
+eval_accumulation_steps:
+    value: null
+eval_dataset:
+    value: true
+eval_delay:
+    value: 0
+eval_do_concat_batches:
+    value: true
+eval_on_start:
+    value: false
+eval_steps:
+    value: 500
+eval_strategy:
+    value: steps
+eval_use_gather_object:
+    value: false
+fp16:
+    value: false
+fp16_backend:
+    value: auto
+fp16_full_eval:
+    value: false
+fp16_opt_level:
+    value: O1
+freeze_protein_model:
+    value: true
+freeze_text_model:
+    value: true
+fsdp:
+    value: []
+fsdp_config:
+    value:
+        min_num_params: 0
+        xla: false
+        xla_fsdp_grad_ckpt: false
+        xla_fsdp_v2: false
+fsdp_min_num_params:
+    value: 0
+fsdp_transformer_layer_cls_to_wrap:
+    value: null
+full_determinism:
+    value: false
+gradient_accumulation_steps:
+    value: 1
+gradient_checkpointing:
+    value: false
+gradient_checkpointing_kwargs:
+    value: null
+greater_is_better:
+    value: true
+group_by_length:
+    value: false
+half_precision_backend:
+    value: auto
+hub_always_push:
+    value: false
+hub_model_id:
+    value: null
+hub_private_repo:
+    value: null
+hub_revision:
+    value: null
+hub_strategy:
+    value: every_save
+hub_token:
+    value: <HUB_TOKEN>
+ignore_data_skip:
+    value: false
+include_for_metrics:
+    value: []
+include_inputs_for_metrics:
+    value: false
+include_num_input_tokens_seen:
+    value: false
+include_tokens_per_second:
+    value: false
+jit_mode_eval:
+    value: false
+label_names:
+    value: null
+label_smoothing_factor:
+    value: 0
+learning_rate:
+    value: 0.0001
+length_column_name:
+    value: length
+liger_kernel_config:
+    value: null
+load_best_model_at_end:
+    value: true
+local_rank:
+    value: 0
+log_level:
+    value: passive
+log_level_replica:
+    value: warning
+log_on_each_node:
+    value: true
+logging_dir:
+    value: ./contrastive_outputs/runs/Aug13_14-56-41_dsw-265304-585cc9d768-ckfd7
+logging_first_step:
+    value: false
+logging_nan_inf_filter:
+    value: true
+logging_steps:
+    value: 100
+logging_strategy:
+    value: steps
+lr_scheduler_type:
+    value: linear
+max_grad_norm:
+    value: 1
+max_length_protein:
+    value: 1024
+max_length_text:
+    value: 512
+max_steps:
+    value: -1
+metric_for_best_model:
+    value: eval_avg_recall_at_1
+mp_parameters:
+    value: ""
+neftune_noise_alpha:
+    value: null
+no_cuda:
+    value: false
+num_epochs:
+    value: 10
+num_query_tokens:
+    value: 8
+num_train_epochs:
+    value: 10
+num_workers:
+    value: 8
+optim:
+    value: adamw_torch
+optim_args:
+    value: null
+optim_target_modules:
+    value: null
+output_dir:
+    value: ./contrastive_outputs
+overwrite_output_dir:
+    value: false
+past_index:
+    value: -1
+per_device_eval_batch_size:
+    value: 8
+per_device_train_batch_size:
+    value: 8
+per_gpu_eval_batch_size:
+    value: null
+per_gpu_train_batch_size:
+    value: null
+prediction_loss_only:
+    value: false
+protein_model_name:
+    value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m
+protein_weight:
+    value: 1
+ptm_weight:
+    value: 1
+push_to_hub:
+    value: false
+push_to_hub_model_id:
+    value: null
+push_to_hub_organization:
+    value: null
+push_to_hub_token:
+    value: <PUSH_TO_HUB_TOKEN>
+qformer_model_name:
+    value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft
+ray_scope:
+    value: last
+remove_unused_columns:
+    value: false
+report_to:
+    value:
+        - wandb
+restore_callback_states_from_checkpoint:
+    value: false
+resume_from_checkpoint:
+    value: null
+run_name:
+    value: null
+save_on_each_node:
+    value: false
+save_only_model:
+    value: false
+save_safetensors:
+    value: true
+save_steps:
+    value: 1000
+save_strategy:
+    value: steps
+save_total_limit:
+    value: 3
+seed:
+    value: 42
+skip_memory_metrics:
+    value: true
+temperature:
+    value: 0.07
+text_model_name:
+    value: /oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged
+text_weight:
+    value: 1
+tf32:
+    value: null
+torch_compile:
+    value: false
+torch_compile_backend:
+    value: null
+torch_compile_mode:
+    value: null
+torch_empty_cache_steps:
+    value: null
+torchdynamo:
+    value: null
+tpu_metrics_debug:
+    value: false
+tpu_num_cores:
+    value: null
+train_dataset:
+    value: /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl
+use_cpu:
+    value: false
+use_ipex:
+    value: false
+use_legacy_prediction_loop:
+    value: false
+use_liger_kernel:
+    value: false
+use_mps_device:
+    value: false
+use_wandb:
+    value: true
+valid_dataset:
+    value: /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl
+wandb_entity:
+    value: null
+wandb_project:
+    value: protein-llm-contrastive
+warmup_ratio:
+    value: 0
+warmup_steps:
+    value: 1000
+weight_decay:
+    value: 0.01

BioReason_new/wandb/run-20250813_145627-ep0ib6n2/files/output.log

@@ -0,0 +1,39 @@
+Loading model...
+Loading checkpoint shards: 100%|██████████████████████████████████████████████████████| 4/4 [00:01<00:00,  2.46it/s]
+Some weights of EsmModel were not initialized from the model checkpoint at /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m and are newly initialized: ['pooler.dense.bias', 'pooler.dense.weight']
+You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
+Loading datasets...
+Training dataset size: 430595
+Eval dataset size: 10000
+Initializing enhanced trainer...
+[2025-08-13 14:56:45,813] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/_distutils_hack/__init__.py:53: UserWarning: Reliance on distutils from stdlib is deprecated. Users must rely on setuptools to provide the distutils module. Avoid importing distutils or import setuptools first, and avoid setting SETUPTOOLS_USE_DISTUTILS=stdlib. Register concerns at https://github.com/pypa/setuptools/issues/new?template=distutils-deprecation.yml
+  warnings.warn(
+[2025-08-13 14:56:46,768] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False
+Starting enhanced contrastive training...
+- Contrastive learning enabled
+- Protein-text matching: enabled
+- Temperature: 0.07
+- PTM weight: 1.0
+  0%|                                                                                     | 0/67290 [00:00<?, ?it/s]Traceback (most recent call last):
+  File "/nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py", line 552, in <module>
+    trainer = main(args)
+              ^^^^^^^^^^
+  File "/nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py", line 399, in main
+    trainer.train()
+  File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/trainer.py", line 2238, in train
+    return inner_training_loop(
+           ^^^^^^^^^^^^^^^^^^^^
+  File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/trainer.py", line 2582, in _inner_training_loop
+    tr_loss_step = self.training_step(model, inputs, num_items_in_batch)
+                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/trainer.py", line 3796, in training_step
+    loss = self.compute_loss(model, inputs, num_items_in_batch=num_items_in_batch)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/nas/shared/kilab/wangyujia/BioReason_new/bioreason/trainer/contrast_trainer_new.py", line 303, in compute_loss
+    sim_p2t, sim_t2p, loss_contrastive = self.contrastive_loss.contrast_global(
+                                         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/nas/shared/kilab/wangyujia/BioReason_new/bioreason/trainer/contrast_trainer_new.py", line 114, in contrast_global
+    sim_t2p = (features_text.unsqueeze(1).unsqueeze(1) @ features_protein_all.permute(0, 2, 1)).squeeze()
+                                                         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+RuntimeError: permute(sparse_coo): number of dimensions in the tensor input does not match the length of the desired ordering of dimensions i.e. input.dim() = 2 is not equal to len(dims) = 3

BioReason_new/wandb/run-20250813_145627-ep0ib6n2/files/requirements.txt

@@ -0,0 +1,233 @@
+jupyterlab_pygments==0.3.0
+depyf==0.19.0
+av==15.0.0
+fastapi==0.116.1
+ray==2.48.0
+cachetools==6.1.0
+prometheus_client==0.22.1
+requests==2.32.4
+jupyter-lsp==2.2.6
+jupyter-events==0.12.0
+anyio==4.10.0
+pybase64==1.4.2
+fastjsonschema==2.21.1
+cloudpickle==3.1.1
+sentry-sdk==2.34.1
+torchmetrics==1.8.1
+wheel==0.45.1
+idna==3.10
+defusedxml==0.7.1
+starlette==0.47.2
+python-dotenv==1.1.1
+comm==0.2.3
+multidict==6.6.3
+python-json-logger==3.3.0
+jupyter_server==2.16.0
+nvidia-nvtx-cu12==12.6.77
+websockets==15.0.1
+referencing==0.36.2
+gguf==0.17.1
+lm-format-enforcer==0.10.12
+xformers==0.0.31
+ninja==1.11.1.4
+beautifulsoup4==4.13.4
+dnspython==2.7.0
+certifi==2025.8.3
+markdown-it-py==3.0.0
+stack-data==0.6.3
+nvidia-cusparselt-cu12==0.6.3
+networkx==3.5
+aiohappyeyeballs==2.6.1
+nvidia-cuda-cupti-cu12==12.6.80
+llguidance==0.7.30
+rich==14.1.0
+outlines_core==0.2.10
+nvidia-nccl-cu12==2.26.2
+pandas==2.3.1
+pydantic_core==2.33.2
+argon2-cffi==25.1.0
+babel==2.17.0
+isoduration==20.11.0
+uvloop==0.21.0
+json5==0.12.0
+widgetsnbextension==4.0.14
+aiohttp==3.12.15
+notebook_shim==0.2.4
+pandocfilters==1.5.1
+vllm==0.10.0
+blake3==1.0.5
+diskcache==5.6.3
+h11==0.16.0
+attrs==25.3.0
+nvidia-cusolver-cu12==11.7.1.2
+click==8.2.1
+cupy-cuda12x==13.5.1
+jedi==0.19.2
+py-cpuinfo==9.0.0
+overrides==7.7.0
+pytz==2025.2
+protobuf==6.31.1
+dill==0.3.8
+ipykernel==6.30.1
+frozenlist==1.7.0
+huggingface-hub==0.34.4
+rignore==0.6.4
+mistune==3.1.3
+sniffio==1.3.1
+httpx==0.28.1
+sentencepiece==0.2.0
+typing_extensions==4.14.1
+typing-inspection==0.4.1
+async-lru==2.0.5
+bioreason==0.1.0
+charset-normalizer==3.4.3
+llvmlite==0.44.0
+safetensors==0.6.2
+fqdn==1.5.1
+peft==0.17.0
+fastapi-cli==0.0.8
+torch==2.7.1
+jsonschema==4.25.0
+torchvision==0.22.1
+parso==0.8.4
+pillow==11.3.0
+pycountry==24.6.1
+scipy==1.16.1
+wandb==0.21.1
+pip==25.1
+wcwidth==0.2.13
+mistral_common==1.8.3
+nvidia-cudnn-cu12==9.5.1.17
+asttokens==3.0.0
+numpy==2.2.6
+uvicorn==0.35.0
+jupyter_client==8.6.3
+debugpy==1.8.16
+fastapi-cloud-cli==0.1.5
+compressed-tensors==0.10.2
+GitPython==3.1.45
+nvidia-ml-py==13.580.65
+tiktoken==0.11.0
+partial-json-parser==0.2.1.1.post6
+prompt_toolkit==3.0.51
+uri-template==1.3.0
+pyarrow==21.0.0
+nvidia-cuda-nvrtc-cu12==12.6.77
+propcache==0.3.2
+matplotlib-inline==0.1.7
+pure_eval==0.2.3
+nest-asyncio==1.6.0
+deepspeed==0.17.4
+einops==0.8.1
+triton==3.3.1
+trl==0.21.0
+pyzmq==27.0.1
+httptools==0.6.4
+ipython==9.4.0
+packaging==25.0
+hjson==3.1.0
+soxr==0.5.0.post1
+webcolors==24.11.1
+PyYAML==6.0.2
+ipywidgets==8.1.7
+lark==1.2.2
+python-multipart==0.0.20
+tinycss2==1.4.0
+mpmath==1.3.0
+psutil==7.0.0
+nbformat==5.10.4
+qwen-vl-utils==0.0.11
+notebook==7.4.5
+jupyterlab==4.4.5
+transformers==4.55.0
+urllib3==2.5.0
+terminado==0.18.1
+xgrammar==0.1.21
+xxhash==3.5.0
+tzdata==2025.2
+jupyterlab_widgets==3.0.15
+aiosignal==1.4.0
+annotated-types==0.7.0
+nvidia-nvjitlink-cu12==12.6.85
+httpcore==1.0.9
+pexpect==4.9.0
+tornado==6.5.2
+regex==2025.7.33
+filelock==3.18.0
+nvidia-cuda-runtime-cu12==12.6.77
+openai==1.90.0
+Pygments==2.19.2
+cffi==1.17.1
+nvidia-cufft-cu12==11.3.0.4
+rpds-py==0.27.0
+rich-toolkit==0.15.0
+bleach==6.2.0
+sympy==1.14.0
+python-dateutil==2.9.0.post0
+watchfiles==1.1.0
+nbclient==0.10.2
+hf-xet==1.1.7
+Send2Trash==1.8.3
+smmap==5.0.2
+tokenizers==0.21.4
+yarl==1.20.1
+MarkupSafe==3.0.2
+prometheus-fastapi-instrumentator==7.1.0
+rfc3986-validator==0.1.1
+bitsandbytes==0.46.1
+nvidia-cufile-cu12==1.11.1.6
+fsspec==2025.3.0
+executing==2.2.0
+pydantic==2.11.7
+jupyter_core==5.8.1
+msgspec==0.19.0
+nvidia-cusparse-cu12==12.5.4.2
+opencv-python-headless==4.12.0.88
+webencodings==0.5.1
+ptyprocess==0.7.0
+pytorch-lightning==2.5.2
+Jinja2==3.1.6
+websocket-client==1.8.0
+jsonpointer==3.0.0
+traitlets==5.14.3
+numba==0.61.2
+soupsieve==2.7
+cbor2==5.6.5
+multiprocess==0.70.16
+email_validator==2.2.0
+distro==1.9.0
+accelerate==1.10.0
+torchaudio==2.7.1
+jiter==0.10.0
+rfc3339-validator==0.1.4
+jupyter==1.1.1
+typer==0.16.0
+gitdb==4.0.12
+types-python-dateutil==2.9.0.20250809
+soundfile==0.13.1
+astor==0.8.1
+shellingham==1.5.4
+jupyterlab_server==2.27.3
+nvidia-curand-cu12==10.3.7.77
+jupyter-console==6.6.3
+msgpack==1.1.1
+six==1.17.0
+pycparser==2.22
+arrow==1.3.0
+tqdm==4.67.1
+lightning-utilities==0.15.2
+interegular==0.3.3
+rfc3987-syntax==1.1.0
+ipython_pygments_lexers==1.1.1
+datasets==4.0.0
+jupyter_server_terminals==0.5.3
+platformdirs==4.3.8
+pydantic-extra-types==2.10.5
+decorator==5.2.1
+mdurl==0.1.2
+nvidia-cublas-cu12==12.6.4.1
+jsonschema-specifications==2025.4.1
+setuptools==78.1.1
+fastrlock==0.8.3
+argon2-cffi-bindings==25.1.0
+nbconvert==7.16.6

BioReason_new/wandb/run-20250813_145627-ep0ib6n2/files/wandb-metadata.json

@@ -0,0 +1,116 @@
+{
+  "os": "Linux-5.10.134-008.18.kangaroo.al8.x86_64-x86_64-with-glibc2.35",
+  "python": "CPython 3.11.0",
+  "startedAt": "2025-08-13T06:56:27.037635Z",
+  "args": [
+    "--text_model_name",
+    "/oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged",
+    "--protein_model_name",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m",
+    "--qformer_model_name",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft",
+    "--num_query_tokens",
+    "8",
+    "--train_dataset",
+    "/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl",
+    "--valid_dataset",
+    "/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl",
+    "--output_dir",
+    "./contrastive_outputs",
+    "--num_epochs",
+    "10",
+    "--batch_size",
+    "8",
+    "--learning_rate",
+    "1e-4",
+    "--temperature",
+    "0.07",
+    "--freeze_protein_model",
+    "--freeze_text_model",
+    "--enable_ptm",
+    "--max_length_protein",
+    "1024",
+    "--max_length_text",
+    "512",
+    "--num_workers",
+    "8",
+    "--eval_dataset",
+    "--use_wandb",
+    "--wandb_project",
+    "protein-llm-contrastive",
+    "--logging_steps",
+    "100",
+    "--eval_steps",
+    "500",
+    "--save_steps",
+    "1000"
+  ],
+  "program": "/nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py",
+  "codePath": "wangyujia/BioReason_new/train_contrastive.py",
+  "codePathLocal": "train_contrastive.py",
+  "git": {
+    "remote": "https://github.com/PorUna-byte/PAR.git",
+    "commit": "b8caf406aa1699c788f0ca6e44a1769452c317db"
+  },
+  "email": "gia0603yucca@gmail.com",
+  "root": "/nas/shared/kilab/wangyujia/BioReason_new",
+  "host": "dsw-265304-585cc9d768-ckfd7",
+  "executable": "/root/miniconda3/envs/bioreason/bin/python",
+  "cpu_count": 64,
+  "cpu_count_logical": 64,
+  "gpu": "NVIDIA A800-SXM4-80GB",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1623302262784",
+      "used": "1015189504"
+    }
+  },
+  "memory": {
+    "total": "549755813888"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-62fcc532-880f-7324-eb13-3b739a38c29b"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-af1197aa-3e7d-a53e-185d-04b1d64164da"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-3e6e551f-2d7d-a27b-654a-da56933d2283"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-a947b64a-c545-ee5e-30e6-b4b0e2dd8fdc"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-ebd04f3f-c437-1b8b-26f2-4b1ebfee93c4"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-53994891-72a6-25be-a4c8-e32c695155b0"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-a927d00a-15c5-31f5-9ae1-9d2e773821a1"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-93d555c4-29a0-cd78-abac-d98cd160d4e1"
+    }
+  ],
+  "cudaVersion": "12.1",
+  "writerId": "8kepep2ysih7kn77h1iftipfg3shqy5q"
+}

BioReason_new/wandb/run-20250813_145627-ep0ib6n2/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":18},"_runtime":18}

BioReason_new/wandb/run-20250813_145627-ep0ib6n2/logs/debug-internal.log

@@ -0,0 +1,39 @@
+{"time":"2025-08-13T14:56:27.316763198+08:00","level":"INFO","msg":"stream: starting","core version":"0.21.1"}
+{"time":"2025-08-13T14:56:30.675423119+08:00","level":"INFO","msg":"stream: created new stream","id":"ep0ib6n2"}
+{"time":"2025-08-13T14:56:30.676242172+08:00","level":"INFO","msg":"stream: started","id":"ep0ib6n2"}
+{"time":"2025-08-13T14:56:30.676275983+08:00","level":"INFO","msg":"sender: started","stream_id":"ep0ib6n2"}
+{"time":"2025-08-13T14:56:30.676265881+08:00","level":"INFO","msg":"writer: started","stream_id":"ep0ib6n2"}
+{"time":"2025-08-13T14:56:30.676313509+08:00","level":"INFO","msg":"handler: started","stream_id":"ep0ib6n2"}
+{"time":"2025-08-13T14:56:35.648007444+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=8a95163e0c0f889892ef15b4a88e43d530288eda1d8571066f8e5a5a448332ea60912856e41297390d209d6ef04c28c92498205a84c3ed6ba3f952108c0e9652e9d14ab6e33add6388a6f39baf85d21db8f98a78b2d267a131a011a5d79b267ba86938c2efe0d867337f5cf8a27924e4ca34fda8e2302d0ddbfa5d507e64601856113db1b9ace6fb3ee615f8f3b05ce7a2fe03f38fd8953c9ed405d6a5ff00bd2c59023b984f8a19d6b9653b17a073d109be52afdedc26e456311889d732a252a242a9388117272376c4cec78ffcef9c8f238bea6b854f7189d3e1e721e175292f5da73859ff0d5f0ac7ee2ca45cad4bb3ea441cfb6d3d91f42f9bf287312182&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:40004->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=8a95163e0c0f889892ef15b4a88e43d530288eda1d8571066f8e5a5a448332ea60912856e41297390d209d6ef04c28c92498205a84c3ed6ba3f952108c0e9652e9d14ab6e33add6388a6f39baf85d21db8f98a78b2d267a131a011a5d79b267ba86938c2efe0d867337f5cf8a27924e4ca34fda8e2302d0ddbfa5d507e64601856113db1b9ace6fb3ee615f8f3b05ce7a2fe03f38fd8953c9ed405d6a5ff00bd2c59023b984f8a19d6b9653b17a073d109be52afdedc26e456311889d732a252a242a9388117272376c4cec78ffcef9c8f238bea6b854f7189d3e1e721e175292f5da73859ff0d5f0ac7ee2ca45cad4bb3ea441cfb6d3d91f42f9bf287312182&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:56:38.348315908+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=8a95163e0c0f889892ef15b4a88e43d530288eda1d8571066f8e5a5a448332ea60912856e41297390d209d6ef04c28c92498205a84c3ed6ba3f952108c0e9652e9d14ab6e33add6388a6f39baf85d21db8f98a78b2d267a131a011a5d79b267ba86938c2efe0d867337f5cf8a27924e4ca34fda8e2302d0ddbfa5d507e64601856113db1b9ace6fb3ee615f8f3b05ce7a2fe03f38fd8953c9ed405d6a5ff00bd2c59023b984f8a19d6b9653b17a073d109be52afdedc26e456311889d732a252a242a9388117272376c4cec78ffcef9c8f238bea6b854f7189d3e1e721e175292f5da73859ff0d5f0ac7ee2ca45cad4bb3ea441cfb6d3d91f42f9bf287312182&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:46218->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=8a95163e0c0f889892ef15b4a88e43d530288eda1d8571066f8e5a5a448332ea60912856e41297390d209d6ef04c28c92498205a84c3ed6ba3f952108c0e9652e9d14ab6e33add6388a6f39baf85d21db8f98a78b2d267a131a011a5d79b267ba86938c2efe0d867337f5cf8a27924e4ca34fda8e2302d0ddbfa5d507e64601856113db1b9ace6fb3ee615f8f3b05ce7a2fe03f38fd8953c9ed405d6a5ff00bd2c59023b984f8a19d6b9653b17a073d109be52afdedc26e456311889d732a252a242a9388117272376c4cec78ffcef9c8f238bea6b854f7189d3e1e721e175292f5da73859ff0d5f0ac7ee2ca45cad4bb3ea441cfb6d3d91f42f9bf287312182&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:56:43.61700136+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=8a95163e0c0f889892ef15b4a88e43d530288eda1d8571066f8e5a5a448332ea60912856e41297390d209d6ef04c28c92498205a84c3ed6ba3f952108c0e9652e9d14ab6e33add6388a6f39baf85d21db8f98a78b2d267a131a011a5d79b267ba86938c2efe0d867337f5cf8a27924e4ca34fda8e2302d0ddbfa5d507e64601856113db1b9ace6fb3ee615f8f3b05ce7a2fe03f38fd8953c9ed405d6a5ff00bd2c59023b984f8a19d6b9653b17a073d109be52afdedc26e456311889d732a252a242a9388117272376c4cec78ffcef9c8f238bea6b854f7189d3e1e721e175292f5da73859ff0d5f0ac7ee2ca45cad4bb3ea441cfb6d3d91f42f9bf287312182&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:46222->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=8a95163e0c0f889892ef15b4a88e43d530288eda1d8571066f8e5a5a448332ea60912856e41297390d209d6ef04c28c92498205a84c3ed6ba3f952108c0e9652e9d14ab6e33add6388a6f39baf85d21db8f98a78b2d267a131a011a5d79b267ba86938c2efe0d867337f5cf8a27924e4ca34fda8e2302d0ddbfa5d507e64601856113db1b9ace6fb3ee615f8f3b05ce7a2fe03f38fd8953c9ed405d6a5ff00bd2c59023b984f8a19d6b9653b17a073d109be52afdedc26e456311889d732a252a242a9388117272376c4cec78ffcef9c8f238bea6b854f7189d3e1e721e175292f5da73859ff0d5f0ac7ee2ca45cad4bb3ea441cfb6d3d91f42f9bf287312182&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:56:51.69459817+08:00","level":"INFO","msg":"stream: closing","id":"ep0ib6n2"}
+{"time":"2025-08-13T14:56:52.777037066+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=8a95163e0c0f889892ef15b4a88e43d530288eda1d8571066f8e5a5a448332ea60912856e41297390d209d6ef04c28c92498205a84c3ed6ba3f952108c0e9652e9d14ab6e33add6388a6f39baf85d21db8f98a78b2d267a131a011a5d79b267ba86938c2efe0d867337f5cf8a27924e4ca34fda8e2302d0ddbfa5d507e64601856113db1b9ace6fb3ee615f8f3b05ce7a2fe03f38fd8953c9ed405d6a5ff00bd2c59023b984f8a19d6b9653b17a073d109be52afdedc26e456311889d732a252a242a9388117272376c4cec78ffcef9c8f238bea6b854f7189d3e1e721e175292f5da73859ff0d5f0ac7ee2ca45cad4bb3ea441cfb6d3d91f42f9bf287312182&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:36500->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=8a95163e0c0f889892ef15b4a88e43d530288eda1d8571066f8e5a5a448332ea60912856e41297390d209d6ef04c28c92498205a84c3ed6ba3f952108c0e9652e9d14ab6e33add6388a6f39baf85d21db8f98a78b2d267a131a011a5d79b267ba86938c2efe0d867337f5cf8a27924e4ca34fda8e2302d0ddbfa5d507e64601856113db1b9ace6fb3ee615f8f3b05ce7a2fe03f38fd8953c9ed405d6a5ff00bd2c59023b984f8a19d6b9653b17a073d109be52afdedc26e456311889d732a252a242a9388117272376c4cec78ffcef9c8f238bea6b854f7189d3e1e721e175292f5da73859ff0d5f0ac7ee2ca45cad4bb3ea441cfb6d3d91f42f9bf287312182&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:58:45.417748957+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=92ae95a068573a7caa07db2b3bcab253490e0ebe38e0fa761f5a93e49b4c4a5d8e2e667590c2c9e17deb189bbaefcf1b894a2011e61c1124d05ae9a8e9bec1f9d63d6def9d54a24b98f45661ad1a17719ea840a41176dc66ac4ed14242c2d74e4ba8600bdddc171e2f3665abd1a0e289176017c0265fe3c49df53270fd495056a1154d7151533eddfd23de83ffc0ec604a59a24e68e2124c9461b71269bfd51f971767d3402294f3a605f7d2c69730216b6b9e3f14a94e38918150fa9ff89bed4ca0449595c0b2392cbd9fd87b07910c18ceaaf32de694b69ee7d3c9845d3c6ff422849de5ae0bc2018170c0533c41df905d8af0f66ea3ba4713d1810814ca00&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:55988->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=92ae95a068573a7caa07db2b3bcab253490e0ebe38e0fa761f5a93e49b4c4a5d8e2e667590c2c9e17deb189bbaefcf1b894a2011e61c1124d05ae9a8e9bec1f9d63d6def9d54a24b98f45661ad1a17719ea840a41176dc66ac4ed14242c2d74e4ba8600bdddc171e2f3665abd1a0e289176017c0265fe3c49df53270fd495056a1154d7151533eddfd23de83ffc0ec604a59a24e68e2124c9461b71269bfd51f971767d3402294f3a605f7d2c69730216b6b9e3f14a94e38918150fa9ff89bed4ca0449595c0b2392cbd9fd87b07910c18ceaaf32de694b69ee7d3c9845d3c6ff422849de5ae0bc2018170c0533c41df905d8af0f66ea3ba4713d1810814ca00&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:59:10.024483291+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065657Z&X-Goog-Expires=86399&X-Goog-Signature=b499547cdb15cac87e48fd391f9ed5dad4148a5e06655bd6e6ad06f7feb1fe9ec68cf9977d5655d6f0d89419803ffdf41be6376849cf43a02f5131adf34a050997e1d8d7cb7ab1fd893c7419d00769e0721e52a462ae922282e98feb03c1a136b4f3432c28f72e3a7f0a08efbcd101fb4dfa14e0959db1d1908cebae7ee00098b30a81916a127012f6418d1a91a93cfdc2f330bd80726446f2715bfe93a3117b6c1792f2067565ad2ab643e19967291f785e76d1c2a8d25402682089fc241b6fd8dbc721d8e51fb830fa0a6a83aabe60a39eaf11c1ffd475a406852af5563e870416b290e5e3e50c867afb12812a5a1433d3630064613b00ac1e56e148b48cd5&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:38146->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065657Z&X-Goog-Expires=86399&X-Goog-Signature=b499547cdb15cac87e48fd391f9ed5dad4148a5e06655bd6e6ad06f7feb1fe9ec68cf9977d5655d6f0d89419803ffdf41be6376849cf43a02f5131adf34a050997e1d8d7cb7ab1fd893c7419d00769e0721e52a462ae922282e98feb03c1a136b4f3432c28f72e3a7f0a08efbcd101fb4dfa14e0959db1d1908cebae7ee00098b30a81916a127012f6418d1a91a93cfdc2f330bd80726446f2715bfe93a3117b6c1792f2067565ad2ab643e19967291f785e76d1c2a8d25402682089fc241b6fd8dbc721d8e51fb830fa0a6a83aabe60a39eaf11c1ffd475a406852af5563e870416b290e5e3e50c867afb12812a5a1433d3630064613b00ac1e56e148b48cd5&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T14:59:12.58298548+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065657Z&X-Goog-Expires=86399&X-Goog-Signature=b499547cdb15cac87e48fd391f9ed5dad4148a5e06655bd6e6ad06f7feb1fe9ec68cf9977d5655d6f0d89419803ffdf41be6376849cf43a02f5131adf34a050997e1d8d7cb7ab1fd893c7419d00769e0721e52a462ae922282e98feb03c1a136b4f3432c28f72e3a7f0a08efbcd101fb4dfa14e0959db1d1908cebae7ee00098b30a81916a127012f6418d1a91a93cfdc2f330bd80726446f2715bfe93a3117b6c1792f2067565ad2ab643e19967291f785e76d1c2a8d25402682089fc241b6fd8dbc721d8e51fb830fa0a6a83aabe60a39eaf11c1ffd475a406852af5563e870416b290e5e3e50c867afb12812a5a1433d3630064613b00ac1e56e148b48cd5&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:38162->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065657Z&X-Goog-Expires=86399&X-Goog-Signature=b499547cdb15cac87e48fd391f9ed5dad4148a5e06655bd6e6ad06f7feb1fe9ec68cf9977d5655d6f0d89419803ffdf41be6376849cf43a02f5131adf34a050997e1d8d7cb7ab1fd893c7419d00769e0721e52a462ae922282e98feb03c1a136b4f3432c28f72e3a7f0a08efbcd101fb4dfa14e0959db1d1908cebae7ee00098b30a81916a127012f6418d1a91a93cfdc2f330bd80726446f2715bfe93a3117b6c1792f2067565ad2ab643e19967291f785e76d1c2a8d25402682089fc241b6fd8dbc721d8e51fb830fa0a6a83aabe60a39eaf11c1ffd475a406852af5563e870416b290e5e3e50c867afb12812a5a1433d3630064613b00ac1e56e148b48cd5&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T15:00:56.440152443+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=92ae95a068573a7caa07db2b3bcab253490e0ebe38e0fa761f5a93e49b4c4a5d8e2e667590c2c9e17deb189bbaefcf1b894a2011e61c1124d05ae9a8e9bec1f9d63d6def9d54a24b98f45661ad1a17719ea840a41176dc66ac4ed14242c2d74e4ba8600bdddc171e2f3665abd1a0e289176017c0265fe3c49df53270fd495056a1154d7151533eddfd23de83ffc0ec604a59a24e68e2124c9461b71269bfd51f971767d3402294f3a605f7d2c69730216b6b9e3f14a94e38918150fa9ff89bed4ca0449595c0b2392cbd9fd87b07910c18ceaaf32de694b69ee7d3c9845d3c6ff422849de5ae0bc2018170c0533c41df905d8af0f66ea3ba4713d1810814ca00&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:59240->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=92ae95a068573a7caa07db2b3bcab253490e0ebe38e0fa761f5a93e49b4c4a5d8e2e667590c2c9e17deb189bbaefcf1b894a2011e61c1124d05ae9a8e9bec1f9d63d6def9d54a24b98f45661ad1a17719ea840a41176dc66ac4ed14242c2d74e4ba8600bdddc171e2f3665abd1a0e289176017c0265fe3c49df53270fd495056a1154d7151533eddfd23de83ffc0ec604a59a24e68e2124c9461b71269bfd51f971767d3402294f3a605f7d2c69730216b6b9e3f14a94e38918150fa9ff89bed4ca0449595c0b2392cbd9fd87b07910c18ceaaf32de694b69ee7d3c9845d3c6ff422849de5ae0bc2018170c0533c41df905d8af0f66ea3ba4713d1810814ca00&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T15:01:21.034779714+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065652Z&X-Goog-Expires=86399&X-Goog-Signature=70e86fac9315b3f1b972a2e965d19160dd68224c29b64e26d1273b1cf0ee0cd71a46b173414db97392275bb488e58c14002348a4e91b03b1a747512a2ab3b841817edc1288805b4177ab9838fb3417705a17bd647a0d9441f2a9d0f1788c3cb687ab8169e5be11b49506785f6ed913ec58a1cabca533c781c70847d44493af27a7f03875ceb771ebef489a0b4e4b5fa84c501f8a784c8f4a0113ea74af2ef4ea840a2a052e33a1f6b3e354761f6966301b00de787bf35a8f259f43f5c6b298ba7ce56f19efec23286c40af589f8d136ba02a4e8c6ed0ec70ae825ed54fcd70fa5acdd75ae8a4d17a34a554fa3b6f3a2482d9462df72ae4cca800cf50f008bb6a&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:53078->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065652Z&X-Goog-Expires=86399&X-Goog-Signature=70e86fac9315b3f1b972a2e965d19160dd68224c29b64e26d1273b1cf0ee0cd71a46b173414db97392275bb488e58c14002348a4e91b03b1a747512a2ab3b841817edc1288805b4177ab9838fb3417705a17bd647a0d9441f2a9d0f1788c3cb687ab8169e5be11b49506785f6ed913ec58a1cabca533c781c70847d44493af27a7f03875ceb771ebef489a0b4e4b5fa84c501f8a784c8f4a0113ea74af2ef4ea840a2a052e33a1f6b3e354761f6966301b00de787bf35a8f259f43f5c6b298ba7ce56f19efec23286c40af589f8d136ba02a4e8c6ed0ec70ae825ed54fcd70fa5acdd75ae8a4d17a34a554fa3b6f3a2482d9462df72ae4cca800cf50f008bb6a&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T15:01:21.04577582+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065652Z&X-Goog-Expires=86399&X-Goog-Signature=90ed12f3fe9c9b10f930fadccdb3bf6688307cfcb66aa11a69d0f96047f0ff09401ed55e9eb0cb66cbd9d383845cd893c0156b96247b225f27a027b4dc63458a22e352ddf0db6cfd76d6737e9a3197d7ed8937a9550a5b8a9256d656be8b242e54fde209064a20f89273fa6fcaa8086fd9c43159c4782be5263a45e453d93b8c02e34e41a6371ac8052670756a97db99e0c5e679fd89f8005910eeed307e69a82bdc2998e602effba1d7522c5d65ac371bd161f447c40092e29f017f7bfd56c41ecb3eff9690ce74b62251fecf37078861335e37aa8d57bd4c2325dbc55a727fbc627b2e68156e3e8e7c70da5b4ef4c0032dfaa57f327bc58f069084d184a500&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:53070->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065652Z&X-Goog-Expires=86399&X-Goog-Signature=90ed12f3fe9c9b10f930fadccdb3bf6688307cfcb66aa11a69d0f96047f0ff09401ed55e9eb0cb66cbd9d383845cd893c0156b96247b225f27a027b4dc63458a22e352ddf0db6cfd76d6737e9a3197d7ed8937a9550a5b8a9256d656be8b242e54fde209064a20f89273fa6fcaa8086fd9c43159c4782be5263a45e453d93b8c02e34e41a6371ac8052670756a97db99e0c5e679fd89f8005910eeed307e69a82bdc2998e602effba1d7522c5d65ac371bd161f447c40092e29f017f7bfd56c41ecb3eff9690ce74b62251fecf37078861335e37aa8d57bd4c2325dbc55a727fbc627b2e68156e3e8e7c70da5b4ef4c0032dfaa57f327bc58f069084d184a500&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T15:01:33.329184944+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=8a95163e0c0f889892ef15b4a88e43d530288eda1d8571066f8e5a5a448332ea60912856e41297390d209d6ef04c28c92498205a84c3ed6ba3f952108c0e9652e9d14ab6e33add6388a6f39baf85d21db8f98a78b2d267a131a011a5d79b267ba86938c2efe0d867337f5cf8a27924e4ca34fda8e2302d0ddbfa5d507e64601856113db1b9ace6fb3ee615f8f3b05ce7a2fe03f38fd8953c9ed405d6a5ff00bd2c59023b984f8a19d6b9653b17a073d109be52afdedc26e456311889d732a252a242a9388117272376c4cec78ffcef9c8f238bea6b854f7189d3e1e721e175292f5da73859ff0d5f0ac7ee2ca45cad4bb3ea441cfb6d3d91f42f9bf287312182&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:54832->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=8a95163e0c0f889892ef15b4a88e43d530288eda1d8571066f8e5a5a448332ea60912856e41297390d209d6ef04c28c92498205a84c3ed6ba3f952108c0e9652e9d14ab6e33add6388a6f39baf85d21db8f98a78b2d267a131a011a5d79b267ba86938c2efe0d867337f5cf8a27924e4ca34fda8e2302d0ddbfa5d507e64601856113db1b9ace6fb3ee615f8f3b05ce7a2fe03f38fd8953c9ed405d6a5ff00bd2c59023b984f8a19d6b9653b17a073d109be52afdedc26e456311889d732a252a242a9388117272376c4cec78ffcef9c8f238bea6b854f7189d3e1e721e175292f5da73859ff0d5f0ac7ee2ca45cad4bb3ea441cfb6d3d91f42f9bf287312182&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T15:03:11.684216561+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=92ae95a068573a7caa07db2b3bcab253490e0ebe38e0fa761f5a93e49b4c4a5d8e2e667590c2c9e17deb189bbaefcf1b894a2011e61c1124d05ae9a8e9bec1f9d63d6def9d54a24b98f45661ad1a17719ea840a41176dc66ac4ed14242c2d74e4ba8600bdddc171e2f3665abd1a0e289176017c0265fe3c49df53270fd495056a1154d7151533eddfd23de83ffc0ec604a59a24e68e2124c9461b71269bfd51f971767d3402294f3a605f7d2c69730216b6b9e3f14a94e38918150fa9ff89bed4ca0449595c0b2392cbd9fd87b07910c18ceaaf32de694b69ee7d3c9845d3c6ff422849de5ae0bc2018170c0533c41df905d8af0f66ea3ba4713d1810814ca00&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:44832->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=92ae95a068573a7caa07db2b3bcab253490e0ebe38e0fa761f5a93e49b4c4a5d8e2e667590c2c9e17deb189bbaefcf1b894a2011e61c1124d05ae9a8e9bec1f9d63d6def9d54a24b98f45661ad1a17719ea840a41176dc66ac4ed14242c2d74e4ba8600bdddc171e2f3665abd1a0e289176017c0265fe3c49df53270fd495056a1154d7151533eddfd23de83ffc0ec604a59a24e68e2124c9461b71269bfd51f971767d3402294f3a605f7d2c69730216b6b9e3f14a94e38918150fa9ff89bed4ca0449595c0b2392cbd9fd87b07910c18ceaaf32de694b69ee7d3c9845d3c6ff422849de5ae0bc2018170c0533c41df905d8af0f66ea3ba4713d1810814ca00&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T15:03:21.627651503+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=92ae95a068573a7caa07db2b3bcab253490e0ebe38e0fa761f5a93e49b4c4a5d8e2e667590c2c9e17deb189bbaefcf1b894a2011e61c1124d05ae9a8e9bec1f9d63d6def9d54a24b98f45661ad1a17719ea840a41176dc66ac4ed14242c2d74e4ba8600bdddc171e2f3665abd1a0e289176017c0265fe3c49df53270fd495056a1154d7151533eddfd23de83ffc0ec604a59a24e68e2124c9461b71269bfd51f971767d3402294f3a605f7d2c69730216b6b9e3f14a94e38918150fa9ff89bed4ca0449595c0b2392cbd9fd87b07910c18ceaaf32de694b69ee7d3c9845d3c6ff422849de5ae0bc2018170c0533c41df905d8af0f66ea3ba4713d1810814ca00&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:45918->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=92ae95a068573a7caa07db2b3bcab253490e0ebe38e0fa761f5a93e49b4c4a5d8e2e667590c2c9e17deb189bbaefcf1b894a2011e61c1124d05ae9a8e9bec1f9d63d6def9d54a24b98f45661ad1a17719ea840a41176dc66ac4ed14242c2d74e4ba8600bdddc171e2f3665abd1a0e289176017c0265fe3c49df53270fd495056a1154d7151533eddfd23de83ffc0ec604a59a24e68e2124c9461b71269bfd51f971767d3402294f3a605f7d2c69730216b6b9e3f14a94e38918150fa9ff89bed4ca0449595c0b2392cbd9fd87b07910c18ceaaf32de694b69ee7d3c9845d3c6ff422849de5ae0bc2018170c0533c41df905d8af0f66ea3ba4713d1810814ca00&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T15:03:32.12823412+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065652Z&X-Goog-Expires=86399&X-Goog-Signature=70e86fac9315b3f1b972a2e965d19160dd68224c29b64e26d1273b1cf0ee0cd71a46b173414db97392275bb488e58c14002348a4e91b03b1a747512a2ab3b841817edc1288805b4177ab9838fb3417705a17bd647a0d9441f2a9d0f1788c3cb687ab8169e5be11b49506785f6ed913ec58a1cabca533c781c70847d44493af27a7f03875ceb771ebef489a0b4e4b5fa84c501f8a784c8f4a0113ea74af2ef4ea840a2a052e33a1f6b3e354761f6966301b00de787bf35a8f259f43f5c6b298ba7ce56f19efec23286c40af589f8d136ba02a4e8c6ed0ec70ae825ed54fcd70fa5acdd75ae8a4d17a34a554fa3b6f3a2482d9462df72ae4cca800cf50f008bb6a&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:48788->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065652Z&X-Goog-Expires=86399&X-Goog-Signature=70e86fac9315b3f1b972a2e965d19160dd68224c29b64e26d1273b1cf0ee0cd71a46b173414db97392275bb488e58c14002348a4e91b03b1a747512a2ab3b841817edc1288805b4177ab9838fb3417705a17bd647a0d9441f2a9d0f1788c3cb687ab8169e5be11b49506785f6ed913ec58a1cabca533c781c70847d44493af27a7f03875ceb771ebef489a0b4e4b5fa84c501f8a784c8f4a0113ea74af2ef4ea840a2a052e33a1f6b3e354761f6966301b00de787bf35a8f259f43f5c6b298ba7ce56f19efec23286c40af589f8d136ba02a4e8c6ed0ec70ae825ed54fcd70fa5acdd75ae8a4d17a34a554fa3b6f3a2482d9462df72ae4cca800cf50f008bb6a&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T15:03:32.147257574+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065652Z&X-Goog-Expires=86399&X-Goog-Signature=90ed12f3fe9c9b10f930fadccdb3bf6688307cfcb66aa11a69d0f96047f0ff09401ed55e9eb0cb66cbd9d383845cd893c0156b96247b225f27a027b4dc63458a22e352ddf0db6cfd76d6737e9a3197d7ed8937a9550a5b8a9256d656be8b242e54fde209064a20f89273fa6fcaa8086fd9c43159c4782be5263a45e453d93b8c02e34e41a6371ac8052670756a97db99e0c5e679fd89f8005910eeed307e69a82bdc2998e602effba1d7522c5d65ac371bd161f447c40092e29f017f7bfd56c41ecb3eff9690ce74b62251fecf37078861335e37aa8d57bd4c2325dbc55a727fbc627b2e68156e3e8e7c70da5b4ef4c0032dfaa57f327bc58f069084d184a500&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:48796->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065652Z&X-Goog-Expires=86399&X-Goog-Signature=90ed12f3fe9c9b10f930fadccdb3bf6688307cfcb66aa11a69d0f96047f0ff09401ed55e9eb0cb66cbd9d383845cd893c0156b96247b225f27a027b4dc63458a22e352ddf0db6cfd76d6737e9a3197d7ed8937a9550a5b8a9256d656be8b242e54fde209064a20f89273fa6fcaa8086fd9c43159c4782be5263a45e453d93b8c02e34e41a6371ac8052670756a97db99e0c5e679fd89f8005910eeed307e69a82bdc2998e602effba1d7522c5d65ac371bd161f447c40092e29f017f7bfd56c41ecb3eff9690ce74b62251fecf37078861335e37aa8d57bd4c2325dbc55a727fbc627b2e68156e3e8e7c70da5b4ef4c0032dfaa57f327bc58f069084d184a500&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T15:03:38.988365667+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=92ae95a068573a7caa07db2b3bcab253490e0ebe38e0fa761f5a93e49b4c4a5d8e2e667590c2c9e17deb189bbaefcf1b894a2011e61c1124d05ae9a8e9bec1f9d63d6def9d54a24b98f45661ad1a17719ea840a41176dc66ac4ed14242c2d74e4ba8600bdddc171e2f3665abd1a0e289176017c0265fe3c49df53270fd495056a1154d7151533eddfd23de83ffc0ec604a59a24e68e2124c9461b71269bfd51f971767d3402294f3a605f7d2c69730216b6b9e3f14a94e38918150fa9ff89bed4ca0449595c0b2392cbd9fd87b07910c18ceaaf32de694b69ee7d3c9845d3c6ff422849de5ae0bc2018170c0533c41df905d8af0f66ea3ba4713d1810814ca00&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:60578->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=92ae95a068573a7caa07db2b3bcab253490e0ebe38e0fa761f5a93e49b4c4a5d8e2e667590c2c9e17deb189bbaefcf1b894a2011e61c1124d05ae9a8e9bec1f9d63d6def9d54a24b98f45661ad1a17719ea840a41176dc66ac4ed14242c2d74e4ba8600bdddc171e2f3665abd1a0e289176017c0265fe3c49df53270fd495056a1154d7151533eddfd23de83ffc0ec604a59a24e68e2124c9461b71269bfd51f971767d3402294f3a605f7d2c69730216b6b9e3f14a94e38918150fa9ff89bed4ca0449595c0b2392cbd9fd87b07910c18ceaaf32de694b69ee7d3c9845d3c6ff422849de5ae0bc2018170c0533c41df905d8af0f66ea3ba4713d1810814ca00&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T15:03:40.306186375+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065657Z&X-Goog-Expires=86399&X-Goog-Signature=b499547cdb15cac87e48fd391f9ed5dad4148a5e06655bd6e6ad06f7feb1fe9ec68cf9977d5655d6f0d89419803ffdf41be6376849cf43a02f5131adf34a050997e1d8d7cb7ab1fd893c7419d00769e0721e52a462ae922282e98feb03c1a136b4f3432c28f72e3a7f0a08efbcd101fb4dfa14e0959db1d1908cebae7ee00098b30a81916a127012f6418d1a91a93cfdc2f330bd80726446f2715bfe93a3117b6c1792f2067565ad2ab643e19967291f785e76d1c2a8d25402682089fc241b6fd8dbc721d8e51fb830fa0a6a83aabe60a39eaf11c1ffd475a406852af5563e870416b290e5e3e50c867afb12812a5a1433d3630064613b00ac1e56e148b48cd5&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:45266->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065657Z&X-Goog-Expires=86399&X-Goog-Signature=b499547cdb15cac87e48fd391f9ed5dad4148a5e06655bd6e6ad06f7feb1fe9ec68cf9977d5655d6f0d89419803ffdf41be6376849cf43a02f5131adf34a050997e1d8d7cb7ab1fd893c7419d00769e0721e52a462ae922282e98feb03c1a136b4f3432c28f72e3a7f0a08efbcd101fb4dfa14e0959db1d1908cebae7ee00098b30a81916a127012f6418d1a91a93cfdc2f330bd80726446f2715bfe93a3117b6c1792f2067565ad2ab643e19967291f785e76d1c2a8d25402682089fc241b6fd8dbc721d8e51fb830fa0a6a83aabe60a39eaf11c1ffd475a406852af5563e870416b290e5e3e50c867afb12812a5a1433d3630064613b00ac1e56e148b48cd5&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T15:04:17.028708323+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=92ae95a068573a7caa07db2b3bcab253490e0ebe38e0fa761f5a93e49b4c4a5d8e2e667590c2c9e17deb189bbaefcf1b894a2011e61c1124d05ae9a8e9bec1f9d63d6def9d54a24b98f45661ad1a17719ea840a41176dc66ac4ed14242c2d74e4ba8600bdddc171e2f3665abd1a0e289176017c0265fe3c49df53270fd495056a1154d7151533eddfd23de83ffc0ec604a59a24e68e2124c9461b71269bfd51f971767d3402294f3a605f7d2c69730216b6b9e3f14a94e38918150fa9ff89bed4ca0449595c0b2392cbd9fd87b07910c18ceaaf32de694b69ee7d3c9845d3c6ff422849de5ae0bc2018170c0533c41df905d8af0f66ea3ba4713d1810814ca00&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:57084->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=92ae95a068573a7caa07db2b3bcab253490e0ebe38e0fa761f5a93e49b4c4a5d8e2e667590c2c9e17deb189bbaefcf1b894a2011e61c1124d05ae9a8e9bec1f9d63d6def9d54a24b98f45661ad1a17719ea840a41176dc66ac4ed14242c2d74e4ba8600bdddc171e2f3665abd1a0e289176017c0265fe3c49df53270fd495056a1154d7151533eddfd23de83ffc0ec604a59a24e68e2124c9461b71269bfd51f971767d3402294f3a605f7d2c69730216b6b9e3f14a94e38918150fa9ff89bed4ca0449595c0b2392cbd9fd87b07910c18ceaaf32de694b69ee7d3c9845d3c6ff422849de5ae0bc2018170c0533c41df905d8af0f66ea3ba4713d1810814ca00&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T15:05:17.273158453+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=92ae95a068573a7caa07db2b3bcab253490e0ebe38e0fa761f5a93e49b4c4a5d8e2e667590c2c9e17deb189bbaefcf1b894a2011e61c1124d05ae9a8e9bec1f9d63d6def9d54a24b98f45661ad1a17719ea840a41176dc66ac4ed14242c2d74e4ba8600bdddc171e2f3665abd1a0e289176017c0265fe3c49df53270fd495056a1154d7151533eddfd23de83ffc0ec604a59a24e68e2124c9461b71269bfd51f971767d3402294f3a605f7d2c69730216b6b9e3f14a94e38918150fa9ff89bed4ca0449595c0b2392cbd9fd87b07910c18ceaaf32de694b69ee7d3c9845d3c6ff422849de5ae0bc2018170c0533c41df905d8af0f66ea3ba4713d1810814ca00&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:43660->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=92ae95a068573a7caa07db2b3bcab253490e0ebe38e0fa761f5a93e49b4c4a5d8e2e667590c2c9e17deb189bbaefcf1b894a2011e61c1124d05ae9a8e9bec1f9d63d6def9d54a24b98f45661ad1a17719ea840a41176dc66ac4ed14242c2d74e4ba8600bdddc171e2f3665abd1a0e289176017c0265fe3c49df53270fd495056a1154d7151533eddfd23de83ffc0ec604a59a24e68e2124c9461b71269bfd51f971767d3402294f3a605f7d2c69730216b6b9e3f14a94e38918150fa9ff89bed4ca0449595c0b2392cbd9fd87b07910c18ceaaf32de694b69ee7d3c9845d3c6ff422849de5ae0bc2018170c0533c41df905d8af0f66ea3ba4713d1810814ca00&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T15:05:47.363164165+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065652Z&X-Goog-Expires=86399&X-Goog-Signature=70e86fac9315b3f1b972a2e965d19160dd68224c29b64e26d1273b1cf0ee0cd71a46b173414db97392275bb488e58c14002348a4e91b03b1a747512a2ab3b841817edc1288805b4177ab9838fb3417705a17bd647a0d9441f2a9d0f1788c3cb687ab8169e5be11b49506785f6ed913ec58a1cabca533c781c70847d44493af27a7f03875ceb771ebef489a0b4e4b5fa84c501f8a784c8f4a0113ea74af2ef4ea840a2a052e33a1f6b3e354761f6966301b00de787bf35a8f259f43f5c6b298ba7ce56f19efec23286c40af589f8d136ba02a4e8c6ed0ec70ae825ed54fcd70fa5acdd75ae8a4d17a34a554fa3b6f3a2482d9462df72ae4cca800cf50f008bb6a&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:50704->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065652Z&X-Goog-Expires=86399&X-Goog-Signature=70e86fac9315b3f1b972a2e965d19160dd68224c29b64e26d1273b1cf0ee0cd71a46b173414db97392275bb488e58c14002348a4e91b03b1a747512a2ab3b841817edc1288805b4177ab9838fb3417705a17bd647a0d9441f2a9d0f1788c3cb687ab8169e5be11b49506785f6ed913ec58a1cabca533c781c70847d44493af27a7f03875ceb771ebef489a0b4e4b5fa84c501f8a784c8f4a0113ea74af2ef4ea840a2a052e33a1f6b3e354761f6966301b00de787bf35a8f259f43f5c6b298ba7ce56f19efec23286c40af589f8d136ba02a4e8c6ed0ec70ae825ed54fcd70fa5acdd75ae8a4d17a34a554fa3b6f3a2482d9462df72ae4cca800cf50f008bb6a&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T15:05:57.828717478+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065652Z&X-Goog-Expires=86399&X-Goog-Signature=70e86fac9315b3f1b972a2e965d19160dd68224c29b64e26d1273b1cf0ee0cd71a46b173414db97392275bb488e58c14002348a4e91b03b1a747512a2ab3b841817edc1288805b4177ab9838fb3417705a17bd647a0d9441f2a9d0f1788c3cb687ab8169e5be11b49506785f6ed913ec58a1cabca533c781c70847d44493af27a7f03875ceb771ebef489a0b4e4b5fa84c501f8a784c8f4a0113ea74af2ef4ea840a2a052e33a1f6b3e354761f6966301b00de787bf35a8f259f43f5c6b298ba7ce56f19efec23286c40af589f8d136ba02a4e8c6ed0ec70ae825ed54fcd70fa5acdd75ae8a4d17a34a554fa3b6f3a2482d9462df72ae4cca800cf50f008bb6a&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:40180->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065652Z&X-Goog-Expires=86399&X-Goog-Signature=70e86fac9315b3f1b972a2e965d19160dd68224c29b64e26d1273b1cf0ee0cd71a46b173414db97392275bb488e58c14002348a4e91b03b1a747512a2ab3b841817edc1288805b4177ab9838fb3417705a17bd647a0d9441f2a9d0f1788c3cb687ab8169e5be11b49506785f6ed913ec58a1cabca533c781c70847d44493af27a7f03875ceb771ebef489a0b4e4b5fa84c501f8a784c8f4a0113ea74af2ef4ea840a2a052e33a1f6b3e354761f6966301b00de787bf35a8f259f43f5c6b298ba7ce56f19efec23286c40af589f8d136ba02a4e8c6ed0ec70ae825ed54fcd70fa5acdd75ae8a4d17a34a554fa3b6f3a2482d9462df72ae4cca800cf50f008bb6a&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T15:06:16.74966428+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065652Z&X-Goog-Expires=86399&X-Goog-Signature=70e86fac9315b3f1b972a2e965d19160dd68224c29b64e26d1273b1cf0ee0cd71a46b173414db97392275bb488e58c14002348a4e91b03b1a747512a2ab3b841817edc1288805b4177ab9838fb3417705a17bd647a0d9441f2a9d0f1788c3cb687ab8169e5be11b49506785f6ed913ec58a1cabca533c781c70847d44493af27a7f03875ceb771ebef489a0b4e4b5fa84c501f8a784c8f4a0113ea74af2ef4ea840a2a052e33a1f6b3e354761f6966301b00de787bf35a8f259f43f5c6b298ba7ce56f19efec23286c40af589f8d136ba02a4e8c6ed0ec70ae825ed54fcd70fa5acdd75ae8a4d17a34a554fa3b6f3a2482d9462df72ae4cca800cf50f008bb6a&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:48620->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065652Z&X-Goog-Expires=86399&X-Goog-Signature=70e86fac9315b3f1b972a2e965d19160dd68224c29b64e26d1273b1cf0ee0cd71a46b173414db97392275bb488e58c14002348a4e91b03b1a747512a2ab3b841817edc1288805b4177ab9838fb3417705a17bd647a0d9441f2a9d0f1788c3cb687ab8169e5be11b49506785f6ed913ec58a1cabca533c781c70847d44493af27a7f03875ceb771ebef489a0b4e4b5fa84c501f8a784c8f4a0113ea74af2ef4ea840a2a052e33a1f6b3e354761f6966301b00de787bf35a8f259f43f5c6b298ba7ce56f19efec23286c40af589f8d136ba02a4e8c6ed0ec70ae825ed54fcd70fa5acdd75ae8a4d17a34a554fa3b6f3a2482d9462df72ae4cca800cf50f008bb6a&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T15:06:28.340218333+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=8a95163e0c0f889892ef15b4a88e43d530288eda1d8571066f8e5a5a448332ea60912856e41297390d209d6ef04c28c92498205a84c3ed6ba3f952108c0e9652e9d14ab6e33add6388a6f39baf85d21db8f98a78b2d267a131a011a5d79b267ba86938c2efe0d867337f5cf8a27924e4ca34fda8e2302d0ddbfa5d507e64601856113db1b9ace6fb3ee615f8f3b05ce7a2fe03f38fd8953c9ed405d6a5ff00bd2c59023b984f8a19d6b9653b17a073d109be52afdedc26e456311889d732a252a242a9388117272376c4cec78ffcef9c8f238bea6b854f7189d3e1e721e175292f5da73859ff0d5f0ac7ee2ca45cad4bb3ea441cfb6d3d91f42f9bf287312182&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:57732->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=8a95163e0c0f889892ef15b4a88e43d530288eda1d8571066f8e5a5a448332ea60912856e41297390d209d6ef04c28c92498205a84c3ed6ba3f952108c0e9652e9d14ab6e33add6388a6f39baf85d21db8f98a78b2d267a131a011a5d79b267ba86938c2efe0d867337f5cf8a27924e4ca34fda8e2302d0ddbfa5d507e64601856113db1b9ace6fb3ee615f8f3b05ce7a2fe03f38fd8953c9ed405d6a5ff00bd2c59023b984f8a19d6b9653b17a073d109be52afdedc26e456311889d732a252a242a9388117272376c4cec78ffcef9c8f238bea6b854f7189d3e1e721e175292f5da73859ff0d5f0ac7ee2ca45cad4bb3ea441cfb6d3d91f42f9bf287312182&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T15:07:58.387973824+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065652Z&X-Goog-Expires=86399&X-Goog-Signature=90ed12f3fe9c9b10f930fadccdb3bf6688307cfcb66aa11a69d0f96047f0ff09401ed55e9eb0cb66cbd9d383845cd893c0156b96247b225f27a027b4dc63458a22e352ddf0db6cfd76d6737e9a3197d7ed8937a9550a5b8a9256d656be8b242e54fde209064a20f89273fa6fcaa8086fd9c43159c4782be5263a45e453d93b8c02e34e41a6371ac8052670756a97db99e0c5e679fd89f8005910eeed307e69a82bdc2998e602effba1d7522c5d65ac371bd161f447c40092e29f017f7bfd56c41ecb3eff9690ce74b62251fecf37078861335e37aa8d57bd4c2325dbc55a727fbc627b2e68156e3e8e7c70da5b4ef4c0032dfaa57f327bc58f069084d184a500&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:47722->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065652Z&X-Goog-Expires=86399&X-Goog-Signature=90ed12f3fe9c9b10f930fadccdb3bf6688307cfcb66aa11a69d0f96047f0ff09401ed55e9eb0cb66cbd9d383845cd893c0156b96247b225f27a027b4dc63458a22e352ddf0db6cfd76d6737e9a3197d7ed8937a9550a5b8a9256d656be8b242e54fde209064a20f89273fa6fcaa8086fd9c43159c4782be5263a45e453d93b8c02e34e41a6371ac8052670756a97db99e0c5e679fd89f8005910eeed307e69a82bdc2998e602effba1d7522c5d65ac371bd161f447c40092e29f017f7bfd56c41ecb3eff9690ce74b62251fecf37078861335e37aa8d57bd4c2325dbc55a727fbc627b2e68156e3e8e7c70da5b4ef4c0032dfaa57f327bc58f069084d184a500&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T15:08:08.206013764+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065652Z&X-Goog-Expires=86399&X-Goog-Signature=90ed12f3fe9c9b10f930fadccdb3bf6688307cfcb66aa11a69d0f96047f0ff09401ed55e9eb0cb66cbd9d383845cd893c0156b96247b225f27a027b4dc63458a22e352ddf0db6cfd76d6737e9a3197d7ed8937a9550a5b8a9256d656be8b242e54fde209064a20f89273fa6fcaa8086fd9c43159c4782be5263a45e453d93b8c02e34e41a6371ac8052670756a97db99e0c5e679fd89f8005910eeed307e69a82bdc2998e602effba1d7522c5d65ac371bd161f447c40092e29f017f7bfd56c41ecb3eff9690ce74b62251fecf37078861335e37aa8d57bd4c2325dbc55a727fbc627b2e68156e3e8e7c70da5b4ef4c0032dfaa57f327bc58f069084d184a500&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:34432->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065652Z&X-Goog-Expires=86399&X-Goog-Signature=90ed12f3fe9c9b10f930fadccdb3bf6688307cfcb66aa11a69d0f96047f0ff09401ed55e9eb0cb66cbd9d383845cd893c0156b96247b225f27a027b4dc63458a22e352ddf0db6cfd76d6737e9a3197d7ed8937a9550a5b8a9256d656be8b242e54fde209064a20f89273fa6fcaa8086fd9c43159c4782be5263a45e453d93b8c02e34e41a6371ac8052670756a97db99e0c5e679fd89f8005910eeed307e69a82bdc2998e602effba1d7522c5d65ac371bd161f447c40092e29f017f7bfd56c41ecb3eff9690ce74b62251fecf37078861335e37aa8d57bd4c2325dbc55a727fbc627b2e68156e3e8e7c70da5b4ef4c0032dfaa57f327bc58f069084d184a500&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T15:08:10.671647124+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065657Z&X-Goog-Expires=86399&X-Goog-Signature=b499547cdb15cac87e48fd391f9ed5dad4148a5e06655bd6e6ad06f7feb1fe9ec68cf9977d5655d6f0d89419803ffdf41be6376849cf43a02f5131adf34a050997e1d8d7cb7ab1fd893c7419d00769e0721e52a462ae922282e98feb03c1a136b4f3432c28f72e3a7f0a08efbcd101fb4dfa14e0959db1d1908cebae7ee00098b30a81916a127012f6418d1a91a93cfdc2f330bd80726446f2715bfe93a3117b6c1792f2067565ad2ab643e19967291f785e76d1c2a8d25402682089fc241b6fd8dbc721d8e51fb830fa0a6a83aabe60a39eaf11c1ffd475a406852af5563e870416b290e5e3e50c867afb12812a5a1433d3630064613b00ac1e56e148b48cd5&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:56624->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065657Z&X-Goog-Expires=86399&X-Goog-Signature=b499547cdb15cac87e48fd391f9ed5dad4148a5e06655bd6e6ad06f7feb1fe9ec68cf9977d5655d6f0d89419803ffdf41be6376849cf43a02f5131adf34a050997e1d8d7cb7ab1fd893c7419d00769e0721e52a462ae922282e98feb03c1a136b4f3432c28f72e3a7f0a08efbcd101fb4dfa14e0959db1d1908cebae7ee00098b30a81916a127012f6418d1a91a93cfdc2f330bd80726446f2715bfe93a3117b6c1792f2067565ad2ab643e19967291f785e76d1c2a8d25402682089fc241b6fd8dbc721d8e51fb830fa0a6a83aabe60a39eaf11c1ffd475a406852af5563e870416b290e5e3e50c867afb12812a5a1433d3630064613b00ac1e56e148b48cd5&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T15:08:27.082693937+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=92ae95a068573a7caa07db2b3bcab253490e0ebe38e0fa761f5a93e49b4c4a5d8e2e667590c2c9e17deb189bbaefcf1b894a2011e61c1124d05ae9a8e9bec1f9d63d6def9d54a24b98f45661ad1a17719ea840a41176dc66ac4ed14242c2d74e4ba8600bdddc171e2f3665abd1a0e289176017c0265fe3c49df53270fd495056a1154d7151533eddfd23de83ffc0ec604a59a24e68e2124c9461b71269bfd51f971767d3402294f3a605f7d2c69730216b6b9e3f14a94e38918150fa9ff89bed4ca0449595c0b2392cbd9fd87b07910c18ceaaf32de694b69ee7d3c9845d3c6ff422849de5ae0bc2018170c0533c41df905d8af0f66ea3ba4713d1810814ca00&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:55834->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=92ae95a068573a7caa07db2b3bcab253490e0ebe38e0fa761f5a93e49b4c4a5d8e2e667590c2c9e17deb189bbaefcf1b894a2011e61c1124d05ae9a8e9bec1f9d63d6def9d54a24b98f45661ad1a17719ea840a41176dc66ac4ed14242c2d74e4ba8600bdddc171e2f3665abd1a0e289176017c0265fe3c49df53270fd495056a1154d7151533eddfd23de83ffc0ec604a59a24e68e2124c9461b71269bfd51f971767d3402294f3a605f7d2c69730216b6b9e3f14a94e38918150fa9ff89bed4ca0449595c0b2392cbd9fd87b07910c18ceaaf32de694b69ee7d3c9845d3c6ff422849de5ae0bc2018170c0533c41df905d8af0f66ea3ba4713d1810814ca00&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T15:08:27.873237787+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065652Z&X-Goog-Expires=86399&X-Goog-Signature=90ed12f3fe9c9b10f930fadccdb3bf6688307cfcb66aa11a69d0f96047f0ff09401ed55e9eb0cb66cbd9d383845cd893c0156b96247b225f27a027b4dc63458a22e352ddf0db6cfd76d6737e9a3197d7ed8937a9550a5b8a9256d656be8b242e54fde209064a20f89273fa6fcaa8086fd9c43159c4782be5263a45e453d93b8c02e34e41a6371ac8052670756a97db99e0c5e679fd89f8005910eeed307e69a82bdc2998e602effba1d7522c5d65ac371bd161f447c40092e29f017f7bfd56c41ecb3eff9690ce74b62251fecf37078861335e37aa8d57bd4c2325dbc55a727fbc627b2e68156e3e8e7c70da5b4ef4c0032dfaa57f327bc58f069084d184a500&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:36340->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065652Z&X-Goog-Expires=86399&X-Goog-Signature=90ed12f3fe9c9b10f930fadccdb3bf6688307cfcb66aa11a69d0f96047f0ff09401ed55e9eb0cb66cbd9d383845cd893c0156b96247b225f27a027b4dc63458a22e352ddf0db6cfd76d6737e9a3197d7ed8937a9550a5b8a9256d656be8b242e54fde209064a20f89273fa6fcaa8086fd9c43159c4782be5263a45e453d93b8c02e34e41a6371ac8052670756a97db99e0c5e679fd89f8005910eeed307e69a82bdc2998e602effba1d7522c5d65ac371bd161f447c40092e29f017f7bfd56c41ecb3eff9690ce74b62251fecf37078861335e37aa8d57bd4c2325dbc55a727fbc627b2e68156e3e8e7c70da5b4ef4c0032dfaa57f327bc58f069084d184a500&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-13T15:09:27.468530265+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-08-13T15:09:33.010674188+08:00","level":"INFO","msg":"handler: closed","stream_id":"ep0ib6n2"}
+{"time":"2025-08-13T15:09:33.015821202+08:00","level":"INFO","msg":"sender: closed","stream_id":"ep0ib6n2"}
+{"time":"2025-08-13T15:09:33.015855038+08:00","level":"INFO","msg":"stream: closed","id":"ep0ib6n2"}

BioReason_new/wandb/run-20250813_145627-ep0ib6n2/logs/debug.log

@@ -0,0 +1,23 @@
+2025-08-13 14:56:27,097 INFO    MainThread:10150 [wandb_setup.py:_flush():80] Current SDK version is 0.21.1
+2025-08-13 14:56:27,098 INFO    MainThread:10150 [wandb_setup.py:_flush():80] Configure stats pid to 10150
+2025-08-13 14:56:27,098 INFO    MainThread:10150 [wandb_setup.py:_flush():80] Loading settings from /root/.config/wandb/settings
+2025-08-13 14:56:27,098 INFO    MainThread:10150 [wandb_setup.py:_flush():80] Loading settings from /nas/shared/kilab/wangyujia/BioReason_new/wandb/settings
+2025-08-13 14:56:27,098 INFO    MainThread:10150 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-08-13 14:56:27,098 INFO    MainThread:10150 [wandb_init.py:setup_run_log_directory():703] Logging user logs to /nas/shared/kilab/wangyujia/BioReason_new/wandb/run-20250813_145627-ep0ib6n2/logs/debug.log
+2025-08-13 14:56:27,098 INFO    MainThread:10150 [wandb_init.py:setup_run_log_directory():704] Logging internal logs to /nas/shared/kilab/wangyujia/BioReason_new/wandb/run-20250813_145627-ep0ib6n2/logs/debug-internal.log
+2025-08-13 14:56:27,098 INFO    MainThread:10150 [wandb_init.py:init():830] calling init triggers
+2025-08-13 14:56:27,098 INFO    MainThread:10150 [wandb_init.py:init():835] wandb.init called with sweep_config: {}
+config: {'text_model_name': '/oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged', 'protein_model_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m', 'qformer_model_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft', 'cache_dir': '/model-weights', 'num_query_tokens': 8, 'train_dataset': '/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl', 'valid_dataset': '/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl', 'eval_dataset': True, 'output_dir': './contrastive_outputs', 'num_epochs': 10, 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 0.01, 'warmup_steps': 1000, 'gradient_accumulation_steps': 1, 'temperature': 0.07, 'freeze_protein_model': True, 'freeze_text_model': True, 'protein_weight': 1.0, 'text_weight': 1.0, 'enable_ptm': True, 'ptm_weight': 1.0, 'max_length_protein': 1024, 'max_length_text': 512, 'num_workers': 8, 'logging_steps': 100, 'eval_steps': 500, 'save_steps': 1000, 'save_total_limit': 3, 'fp16': False, 'bf16': False, 'seed': 42, 'use_wandb': True, 'wandb_project': 'protein-llm-contrastive', 'wandb_entity': None, '_wandb': {}}
+2025-08-13 14:56:27,098 INFO    MainThread:10150 [wandb_init.py:init():871] starting backend
+2025-08-13 14:56:27,306 INFO    MainThread:10150 [wandb_init.py:init():874] sending inform_init request
+2025-08-13 14:56:27,312 INFO    MainThread:10150 [wandb_init.py:init():882] backend started and connected
+2025-08-13 14:56:27,313 INFO    MainThread:10150 [wandb_init.py:init():953] updated telemetry
+2025-08-13 14:56:27,341 INFO    MainThread:10150 [wandb_init.py:init():977] communicating run to backend with 90.0 second timeout
+2025-08-13 14:56:33,672 INFO    MainThread:10150 [wandb_init.py:init():1029] starting run threads in backend
+2025-08-13 14:56:33,780 INFO    MainThread:10150 [wandb_run.py:_console_start():2494] atexit reg
+2025-08-13 14:56:33,780 INFO    MainThread:10150 [wandb_run.py:_redirect():2342] redirect: wrap_raw
+2025-08-13 14:56:33,780 INFO    MainThread:10150 [wandb_run.py:_redirect():2411] Wrapping output streams.
+2025-08-13 14:56:33,780 INFO    MainThread:10150 [wandb_run.py:_redirect():2434] Redirects installed.
+2025-08-13 14:56:33,783 INFO    MainThread:10150 [wandb_init.py:init():1075] run started, returning control to user process
+2025-08-13 14:56:47,035 INFO    MainThread:10150 [wandb_run.py:_config_callback():1380] config_cb None None {'output_dir': './contrastive_outputs', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': True, 'do_predict': False, 'eval_strategy': 'steps', 'prediction_loss_only': False, 'per_device_train_batch_size': 8, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'eval_delay': 0, 'torch_empty_cache_steps': None, 'learning_rate': 0.0001, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 10, 'max_steps': -1, 'lr_scheduler_type': 'linear', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.0, 'warmup_steps': 1000, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': './contrastive_outputs/runs/Aug13_14-56-41_dsw-265304-585cc9d768-ckfd7', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 100, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 1000, 'save_total_limit': 3, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'restore_callback_states_from_checkpoint': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 42, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': False, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 500, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': None, 'disable_tqdm': False, 'remove_unused_columns': False, 'label_names': None, 'load_best_model_at_end': True, 'metric_for_best_model': 'eval_avg_recall_at_1', 'greater_is_better': True, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'deepspeed': None, 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['wandb'], 'ddp_find_unused_parameters': False, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_always_push': False, 'hub_revision': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'include_for_metrics': [], 'eval_do_concat_batches': True, 'fp16_backend': 'auto', 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None, 'batch_eval_metrics': False, 'eval_on_start': False, 'use_liger_kernel': False, 'liger_kernel_config': None, 'eval_use_gather_object': False, 'average_tokens_across_devices': False, 'temperature': 0.07, 'freeze_protein_model': True, 'freeze_text_model': True, 'protein_weight': 1.0, 'text_weight': 1.0, 'max_length_protein': 1024, 'max_length_text': 512, 'enable_ptm': True, 'ptm_weight': 1.0}
+2025-08-13 14:56:51,693 INFO    MsgRouterThr:10150 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles.

BioReason_new/wandb/run-20250813_151502-efaj8qxg/files/config.yaml

@@ -0,0 +1,457 @@
+_wandb:
+    value:
+        cli_version: 0.21.1
+        e:
+            wof9vn4farxkwiz68rqknt7azap7s5wq:
+                args:
+                    - --text_model_name
+                    - /oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged
+                    - --protein_model_name
+                    - /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m
+                    - --qformer_model_name
+                    - /nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft
+                    - --num_query_tokens
+                    - "8"
+                    - --train_dataset
+                    - /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl
+                    - --valid_dataset
+                    - /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl
+                    - --output_dir
+                    - ./contrastive_outputs
+                    - --num_epochs
+                    - "10"
+                    - --batch_size
+                    - "8"
+                    - --learning_rate
+                    - "1e-4"
+                    - --temperature
+                    - "0.07"
+                    - --freeze_protein_model
+                    - --freeze_text_model
+                    - --enable_ptm
+                    - --max_length_protein
+                    - "1024"
+                    - --max_length_text
+                    - "512"
+                    - --num_workers
+                    - "8"
+                    - --eval_dataset
+                    - --use_wandb
+                    - --wandb_project
+                    - protein-llm-contrastive
+                    - --logging_steps
+                    - "100"
+                    - --eval_steps
+                    - "500"
+                    - --save_steps
+                    - "1000"
+                codePath: wangyujia/BioReason_new/train_contrastive.py
+                codePathLocal: train_contrastive.py
+                cpu_count: 64
+                cpu_count_logical: 64
+                cudaVersion: "12.1"
+                disk:
+                    /:
+                        total: "1623302262784"
+                        used: "1015259136"
+                executable: /root/miniconda3/envs/bioreason/bin/python
+                git:
+                    commit: b8caf406aa1699c788f0ca6e44a1769452c317db
+                    remote: https://github.com/PorUna-byte/PAR.git
+                gpu: NVIDIA A800-SXM4-80GB
+                gpu_count: 8
+                gpu_nvidia:
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-62fcc532-880f-7324-eb13-3b739a38c29b
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-af1197aa-3e7d-a53e-185d-04b1d64164da
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-3e6e551f-2d7d-a27b-654a-da56933d2283
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-a947b64a-c545-ee5e-30e6-b4b0e2dd8fdc
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-ebd04f3f-c437-1b8b-26f2-4b1ebfee93c4
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-53994891-72a6-25be-a4c8-e32c695155b0
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-a927d00a-15c5-31f5-9ae1-9d2e773821a1
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-93d555c4-29a0-cd78-abac-d98cd160d4e1
+                host: dsw-265304-585cc9d768-ckfd7
+                memory:
+                    total: "549755813888"
+                os: Linux-5.10.134-008.18.kangaroo.al8.x86_64-x86_64-with-glibc2.35
+                program: /nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py
+                python: CPython 3.11.0
+                root: /nas/shared/kilab/wangyujia/BioReason_new
+                startedAt: "2025-08-13T07:15:02.581009Z"
+                writerId: wof9vn4farxkwiz68rqknt7azap7s5wq
+        m:
+            - "1": train/global_step
+              "6":
+                - 3
+              "7": []
+            - "2": '*'
+              "5": 1
+              "6":
+                - 1
+              "7": []
+        python_version: 3.11.0
+        t:
+            "1":
+                - 1
+                - 9
+                - 11
+                - 41
+                - 49
+                - 51
+                - 71
+                - 84
+                - 98
+                - 103
+            "2":
+                - 1
+                - 9
+                - 11
+                - 41
+                - 49
+                - 51
+                - 71
+                - 84
+                - 98
+                - 103
+            "3":
+                - 7
+                - 13
+                - 16
+                - 66
+            "4": 3.11.0
+            "5": 0.21.1
+            "6": 4.55.0
+            "9":
+                "1": transformers_trainer
+            "12": 0.21.1
+            "13": linux-x86_64
+accelerator_config:
+    value:
+        dispatch_batches: null
+        even_batches: true
+        gradient_accumulation_kwargs: null
+        non_blocking: false
+        split_batches: false
+        use_seedable_sampler: true
+adafactor:
+    value: false
+adam_beta1:
+    value: 0.9
+adam_beta2:
+    value: 0.999
+adam_epsilon:
+    value: 1e-08
+auto_find_batch_size:
+    value: false
+average_tokens_across_devices:
+    value: false
+batch_eval_metrics:
+    value: false
+batch_size:
+    value: 8
+bf16:
+    value: false
+bf16_full_eval:
+    value: false
+cache_dir:
+    value: /model-weights
+data_seed:
+    value: null
+dataloader_drop_last:
+    value: false
+dataloader_num_workers:
+    value: 8
+dataloader_persistent_workers:
+    value: false
+dataloader_pin_memory:
+    value: true
+dataloader_prefetch_factor:
+    value: null
+ddp_backend:
+    value: null
+ddp_broadcast_buffers:
+    value: null
+ddp_bucket_cap_mb:
+    value: null
+ddp_find_unused_parameters:
+    value: false
+ddp_timeout:
+    value: 1800
+debug:
+    value: []
+deepspeed:
+    value: null
+disable_tqdm:
+    value: false
+do_eval:
+    value: true
+do_predict:
+    value: false
+do_train:
+    value: false
+enable_ptm:
+    value: true
+eval_accumulation_steps:
+    value: null
+eval_dataset:
+    value: true
+eval_delay:
+    value: 0
+eval_do_concat_batches:
+    value: true
+eval_on_start:
+    value: false
+eval_steps:
+    value: 500
+eval_strategy:
+    value: steps
+eval_use_gather_object:
+    value: false
+fp16:
+    value: false
+fp16_backend:
+    value: auto
+fp16_full_eval:
+    value: false
+fp16_opt_level:
+    value: O1
+freeze_protein_model:
+    value: true
+freeze_text_model:
+    value: true
+fsdp:
+    value: []
+fsdp_config:
+    value:
+        min_num_params: 0
+        xla: false
+        xla_fsdp_grad_ckpt: false
+        xla_fsdp_v2: false
+fsdp_min_num_params:
+    value: 0
+fsdp_transformer_layer_cls_to_wrap:
+    value: null
+full_determinism:
+    value: false
+gradient_accumulation_steps:
+    value: 1
+gradient_checkpointing:
+    value: false
+gradient_checkpointing_kwargs:
+    value: null
+greater_is_better:
+    value: true
+group_by_length:
+    value: false
+half_precision_backend:
+    value: auto
+hub_always_push:
+    value: false
+hub_model_id:
+    value: null
+hub_private_repo:
+    value: null
+hub_revision:
+    value: null
+hub_strategy:
+    value: every_save
+hub_token:
+    value: <HUB_TOKEN>
+ignore_data_skip:
+    value: false
+include_for_metrics:
+    value: []
+include_inputs_for_metrics:
+    value: false
+include_num_input_tokens_seen:
+    value: false
+include_tokens_per_second:
+    value: false
+jit_mode_eval:
+    value: false
+label_names:
+    value: null
+label_smoothing_factor:
+    value: 0
+learning_rate:
+    value: 0.0001
+length_column_name:
+    value: length
+liger_kernel_config:
+    value: null
+load_best_model_at_end:
+    value: true
+local_rank:
+    value: 0
+log_level:
+    value: passive
+log_level_replica:
+    value: warning
+log_on_each_node:
+    value: true
+logging_dir:
+    value: ./contrastive_outputs/runs/Aug13_15-15-23_dsw-265304-585cc9d768-ckfd7
+logging_first_step:
+    value: false
+logging_nan_inf_filter:
+    value: true
+logging_steps:
+    value: 100
+logging_strategy:
+    value: steps
+lr_scheduler_type:
+    value: linear
+max_grad_norm:
+    value: 1
+max_length_protein:
+    value: 1024
+max_length_text:
+    value: 512
+max_steps:
+    value: -1
+metric_for_best_model:
+    value: eval_avg_recall_at_1
+mp_parameters:
+    value: ""
+neftune_noise_alpha:
+    value: null
+no_cuda:
+    value: false
+num_epochs:
+    value: 10
+num_query_tokens:
+    value: 8
+num_train_epochs:
+    value: 10
+num_workers:
+    value: 8
+optim:
+    value: adamw_torch
+optim_args:
+    value: null
+optim_target_modules:
+    value: null
+output_dir:
+    value: ./contrastive_outputs
+overwrite_output_dir:
+    value: false
+past_index:
+    value: -1
+per_device_eval_batch_size:
+    value: 8
+per_device_train_batch_size:
+    value: 8
+per_gpu_eval_batch_size:
+    value: null
+per_gpu_train_batch_size:
+    value: null
+prediction_loss_only:
+    value: false
+protein_model_name:
+    value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m
+protein_weight:
+    value: 1
+ptm_weight:
+    value: 1
+push_to_hub:
+    value: false
+push_to_hub_model_id:
+    value: null
+push_to_hub_organization:
+    value: null
+push_to_hub_token:
+    value: <PUSH_TO_HUB_TOKEN>
+qformer_model_name:
+    value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft
+ray_scope:
+    value: last
+remove_unused_columns:
+    value: false
+report_to:
+    value:
+        - wandb
+restore_callback_states_from_checkpoint:
+    value: false
+resume_from_checkpoint:
+    value: null
+run_name:
+    value: null
+save_on_each_node:
+    value: false
+save_only_model:
+    value: false
+save_safetensors:
+    value: true
+save_steps:
+    value: 1000
+save_strategy:
+    value: steps
+save_total_limit:
+    value: 3
+seed:
+    value: 42
+skip_memory_metrics:
+    value: true
+temperature:
+    value: 0.07
+text_model_name:
+    value: /oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged
+text_weight:
+    value: 1
+tf32:
+    value: null
+torch_compile:
+    value: false
+torch_compile_backend:
+    value: null
+torch_compile_mode:
+    value: null
+torch_empty_cache_steps:
+    value: null
+torchdynamo:
+    value: null
+tpu_metrics_debug:
+    value: false
+tpu_num_cores:
+    value: null
+train_dataset:
+    value: /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl
+use_cpu:
+    value: false
+use_ipex:
+    value: false
+use_legacy_prediction_loop:
+    value: false
+use_liger_kernel:
+    value: false
+use_mps_device:
+    value: false
+use_wandb:
+    value: true
+valid_dataset:
+    value: /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl
+wandb_entity:
+    value: null
+wandb_project:
+    value: protein-llm-contrastive
+warmup_ratio:
+    value: 0
+warmup_steps:
+    value: 1000
+weight_decay:
+    value: 0.01

BioReason_new/wandb/run-20250813_151502-efaj8qxg/files/output.log

@@ -0,0 +1,45 @@
+Loading model...
+Loading checkpoint shards: 100%|██████████████████████████████████████████████████████| 4/4 [00:01<00:00,  2.45it/s]
+Some weights of EsmModel were not initialized from the model checkpoint at /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m and are newly initialized: ['pooler.dense.bias', 'pooler.dense.weight']
+You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
+Loading datasets...
+Training dataset size: 430595
+Eval dataset size: 10000
+Initializing enhanced trainer...
+[2025-08-13 15:15:27,683] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/_distutils_hack/__init__.py:53: UserWarning: Reliance on distutils from stdlib is deprecated. Users must rely on setuptools to provide the distutils module. Avoid importing distutils or import setuptools first, and avoid setting SETUPTOOLS_USE_DISTUTILS=stdlib. Register concerns at https://github.com/pypa/setuptools/issues/new?template=distutils-deprecation.yml
+  warnings.warn(
+[2025-08-13 15:15:28,672] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False
+Starting enhanced contrastive training...
+- Contrastive learning enabled
+- Protein-text matching: enabled
+- Temperature: 0.07
+- PTM weight: 1.0
+  0%|                                                                                     | 0/67290 [00:00<?, ?it/s]Traceback (most recent call last):
+  File "/nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py", line 552, in <module>
+    trainer = main(args)
+              ^^^^^^^^^^
+  File "/nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py", line 399, in main
+    trainer.train()
+  File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/trainer.py", line 2238, in train
+    return inner_training_loop(
+           ^^^^^^^^^^^^^^^^^^^^
+  File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/trainer.py", line 2582, in _inner_training_loop
+    tr_loss_step = self.training_step(model, inputs, num_items_in_batch)
+                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/trainer.py", line 3796, in training_step
+    loss = self.compute_loss(model, inputs, num_items_in_batch=num_items_in_batch)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/nas/shared/kilab/wangyujia/BioReason_new/bioreason/trainer/contrast_trainer_new.py", line 330, in compute_loss
+    loss_ptm = self.contrastive_loss.compute_ptm_loss(
+               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/nas/shared/kilab/wangyujia/BioReason_new/bioreason/trainer/contrast_trainer_new.py", line 196, in compute_ptm_loss
+    output_ptm = qformer(
+                 ^^^^^^^^
+  File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1762, in _call_impl
+    return forward_call(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+TypeError: BertModel.forward() got an unexpected keyword argument 'query_embeds'

BioReason_new/wandb/run-20250813_151502-efaj8qxg/files/requirements.txt

@@ -0,0 +1,233 @@
+jupyterlab_pygments==0.3.0
+depyf==0.19.0
+av==15.0.0
+fastapi==0.116.1
+ray==2.48.0
+cachetools==6.1.0
+prometheus_client==0.22.1
+requests==2.32.4
+jupyter-lsp==2.2.6
+jupyter-events==0.12.0
+anyio==4.10.0
+pybase64==1.4.2
+fastjsonschema==2.21.1
+cloudpickle==3.1.1
+sentry-sdk==2.34.1
+torchmetrics==1.8.1
+wheel==0.45.1
+idna==3.10
+defusedxml==0.7.1
+starlette==0.47.2
+python-dotenv==1.1.1
+comm==0.2.3
+multidict==6.6.3
+python-json-logger==3.3.0
+jupyter_server==2.16.0
+nvidia-nvtx-cu12==12.6.77
+websockets==15.0.1
+referencing==0.36.2
+gguf==0.17.1
+lm-format-enforcer==0.10.12
+xformers==0.0.31
+ninja==1.11.1.4
+beautifulsoup4==4.13.4
+dnspython==2.7.0
+certifi==2025.8.3
+markdown-it-py==3.0.0
+stack-data==0.6.3
+nvidia-cusparselt-cu12==0.6.3
+networkx==3.5
+aiohappyeyeballs==2.6.1
+nvidia-cuda-cupti-cu12==12.6.80
+llguidance==0.7.30
+rich==14.1.0
+outlines_core==0.2.10
+nvidia-nccl-cu12==2.26.2
+pandas==2.3.1
+pydantic_core==2.33.2
+argon2-cffi==25.1.0
+babel==2.17.0
+isoduration==20.11.0
+uvloop==0.21.0
+json5==0.12.0
+widgetsnbextension==4.0.14
+aiohttp==3.12.15
+notebook_shim==0.2.4
+pandocfilters==1.5.1
+vllm==0.10.0
+blake3==1.0.5
+diskcache==5.6.3
+h11==0.16.0
+attrs==25.3.0
+nvidia-cusolver-cu12==11.7.1.2
+click==8.2.1
+cupy-cuda12x==13.5.1
+jedi==0.19.2
+py-cpuinfo==9.0.0
+overrides==7.7.0
+pytz==2025.2
+protobuf==6.31.1
+dill==0.3.8
+ipykernel==6.30.1
+frozenlist==1.7.0
+huggingface-hub==0.34.4
+rignore==0.6.4
+mistune==3.1.3
+sniffio==1.3.1
+httpx==0.28.1
+sentencepiece==0.2.0
+typing_extensions==4.14.1
+typing-inspection==0.4.1
+async-lru==2.0.5
+bioreason==0.1.0
+charset-normalizer==3.4.3
+llvmlite==0.44.0
+safetensors==0.6.2
+fqdn==1.5.1
+peft==0.17.0
+fastapi-cli==0.0.8
+torch==2.7.1
+jsonschema==4.25.0
+torchvision==0.22.1
+parso==0.8.4
+pillow==11.3.0
+pycountry==24.6.1
+scipy==1.16.1
+wandb==0.21.1
+pip==25.1
+wcwidth==0.2.13
+mistral_common==1.8.3
+nvidia-cudnn-cu12==9.5.1.17
+asttokens==3.0.0
+numpy==2.2.6
+uvicorn==0.35.0
+jupyter_client==8.6.3
+debugpy==1.8.16
+fastapi-cloud-cli==0.1.5
+compressed-tensors==0.10.2
+GitPython==3.1.45
+nvidia-ml-py==13.580.65
+tiktoken==0.11.0
+partial-json-parser==0.2.1.1.post6
+prompt_toolkit==3.0.51
+uri-template==1.3.0
+pyarrow==21.0.0
+nvidia-cuda-nvrtc-cu12==12.6.77
+propcache==0.3.2
+matplotlib-inline==0.1.7
+pure_eval==0.2.3
+nest-asyncio==1.6.0
+deepspeed==0.17.4
+einops==0.8.1
+triton==3.3.1
+trl==0.21.0
+pyzmq==27.0.1
+httptools==0.6.4
+ipython==9.4.0
+packaging==25.0
+hjson==3.1.0
+soxr==0.5.0.post1
+webcolors==24.11.1
+PyYAML==6.0.2
+ipywidgets==8.1.7
+lark==1.2.2
+python-multipart==0.0.20
+tinycss2==1.4.0
+mpmath==1.3.0
+psutil==7.0.0
+nbformat==5.10.4
+qwen-vl-utils==0.0.11
+notebook==7.4.5
+jupyterlab==4.4.5
+transformers==4.55.0
+urllib3==2.5.0
+terminado==0.18.1
+xgrammar==0.1.21
+xxhash==3.5.0
+tzdata==2025.2
+jupyterlab_widgets==3.0.15
+aiosignal==1.4.0
+annotated-types==0.7.0
+nvidia-nvjitlink-cu12==12.6.85
+httpcore==1.0.9
+pexpect==4.9.0
+tornado==6.5.2
+regex==2025.7.33
+filelock==3.18.0
+nvidia-cuda-runtime-cu12==12.6.77
+openai==1.90.0
+Pygments==2.19.2
+cffi==1.17.1
+nvidia-cufft-cu12==11.3.0.4
+rpds-py==0.27.0
+rich-toolkit==0.15.0
+bleach==6.2.0
+sympy==1.14.0
+python-dateutil==2.9.0.post0
+watchfiles==1.1.0
+nbclient==0.10.2
+hf-xet==1.1.7
+Send2Trash==1.8.3
+smmap==5.0.2
+tokenizers==0.21.4
+yarl==1.20.1
+MarkupSafe==3.0.2
+prometheus-fastapi-instrumentator==7.1.0
+rfc3986-validator==0.1.1
+bitsandbytes==0.46.1
+nvidia-cufile-cu12==1.11.1.6
+fsspec==2025.3.0
+executing==2.2.0
+pydantic==2.11.7
+jupyter_core==5.8.1
+msgspec==0.19.0
+nvidia-cusparse-cu12==12.5.4.2
+opencv-python-headless==4.12.0.88
+webencodings==0.5.1
+ptyprocess==0.7.0
+pytorch-lightning==2.5.2
+Jinja2==3.1.6
+websocket-client==1.8.0
+jsonpointer==3.0.0
+traitlets==5.14.3
+numba==0.61.2
+soupsieve==2.7
+cbor2==5.6.5
+multiprocess==0.70.16
+email_validator==2.2.0
+distro==1.9.0
+accelerate==1.10.0
+torchaudio==2.7.1
+jiter==0.10.0
+rfc3339-validator==0.1.4
+jupyter==1.1.1
+typer==0.16.0
+gitdb==4.0.12
+types-python-dateutil==2.9.0.20250809
+soundfile==0.13.1
+astor==0.8.1
+shellingham==1.5.4
+jupyterlab_server==2.27.3
+nvidia-curand-cu12==10.3.7.77
+jupyter-console==6.6.3
+msgpack==1.1.1
+six==1.17.0
+pycparser==2.22
+arrow==1.3.0
+tqdm==4.67.1
+lightning-utilities==0.15.2
+interegular==0.3.3
+rfc3987-syntax==1.1.0
+ipython_pygments_lexers==1.1.1
+datasets==4.0.0
+jupyter_server_terminals==0.5.3
+platformdirs==4.3.8
+pydantic-extra-types==2.10.5
+decorator==5.2.1
+mdurl==0.1.2
+nvidia-cublas-cu12==12.6.4.1
+jsonschema-specifications==2025.4.1
+setuptools==78.1.1
+fastrlock==0.8.3
+argon2-cffi-bindings==25.1.0
+nbconvert==7.16.6

BioReason_new/wandb/run-20250813_151502-efaj8qxg/files/wandb-metadata.json

@@ -0,0 +1,115 @@
+{
+  "os": "Linux-5.10.134-008.18.kangaroo.al8.x86_64-x86_64-with-glibc2.35",
+  "python": "CPython 3.11.0",
+  "startedAt": "2025-08-13T07:15:02.581009Z",
+  "args": [
+    "--text_model_name",
+    "/oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged",
+    "--protein_model_name",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m",
+    "--qformer_model_name",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft",
+    "--num_query_tokens",
+    "8",
+    "--train_dataset",
+    "/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl",
+    "--valid_dataset",
+    "/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl",
+    "--output_dir",
+    "./contrastive_outputs",
+    "--num_epochs",
+    "10",
+    "--batch_size",
+    "8",
+    "--learning_rate",
+    "1e-4",
+    "--temperature",
+    "0.07",
+    "--freeze_protein_model",
+    "--freeze_text_model",
+    "--enable_ptm",
+    "--max_length_protein",
+    "1024",
+    "--max_length_text",
+    "512",
+    "--num_workers",
+    "8",
+    "--eval_dataset",
+    "--use_wandb",
+    "--wandb_project",
+    "protein-llm-contrastive",
+    "--logging_steps",
+    "100",
+    "--eval_steps",
+    "500",
+    "--save_steps",
+    "1000"
+  ],
+  "program": "/nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py",
+  "codePath": "wangyujia/BioReason_new/train_contrastive.py",
+  "codePathLocal": "train_contrastive.py",
+  "git": {
+    "remote": "https://github.com/PorUna-byte/PAR.git",
+    "commit": "b8caf406aa1699c788f0ca6e44a1769452c317db"
+  },
+  "root": "/nas/shared/kilab/wangyujia/BioReason_new",
+  "host": "dsw-265304-585cc9d768-ckfd7",
+  "executable": "/root/miniconda3/envs/bioreason/bin/python",
+  "cpu_count": 64,
+  "cpu_count_logical": 64,
+  "gpu": "NVIDIA A800-SXM4-80GB",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1623302262784",
+      "used": "1015259136"
+    }
+  },
+  "memory": {
+    "total": "549755813888"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-62fcc532-880f-7324-eb13-3b739a38c29b"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-af1197aa-3e7d-a53e-185d-04b1d64164da"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-3e6e551f-2d7d-a27b-654a-da56933d2283"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-a947b64a-c545-ee5e-30e6-b4b0e2dd8fdc"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-ebd04f3f-c437-1b8b-26f2-4b1ebfee93c4"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-53994891-72a6-25be-a4c8-e32c695155b0"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-a927d00a-15c5-31f5-9ae1-9d2e773821a1"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "architecture": "Ampere",
+      "uuid": "GPU-93d555c4-29a0-cd78-abac-d98cd160d4e1"
+    }
+  ],
+  "cudaVersion": "12.1",
+  "writerId": "wof9vn4farxkwiz68rqknt7azap7s5wq"
+}

BioReason_new/wandb/run-20250813_151502-efaj8qxg/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":14},"_runtime":14}