diff --git a/BioReason_new/wandb/run-20250813_121906-3wcb4wer/files/requirements.txt b/BioReason_new/wandb/run-20250813_121906-3wcb4wer/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..743d03fe27e0b7e200935023c07a1d7fa61b99ce --- /dev/null +++ b/BioReason_new/wandb/run-20250813_121906-3wcb4wer/files/requirements.txt @@ -0,0 +1,233 @@ +jupyterlab_pygments==0.3.0 +depyf==0.19.0 +av==15.0.0 +fastapi==0.116.1 +ray==2.48.0 +cachetools==6.1.0 +prometheus_client==0.22.1 +requests==2.32.4 +jupyter-lsp==2.2.6 +jupyter-events==0.12.0 +anyio==4.10.0 +pybase64==1.4.2 +fastjsonschema==2.21.1 +cloudpickle==3.1.1 +sentry-sdk==2.34.1 +torchmetrics==1.8.1 +wheel==0.45.1 +idna==3.10 +defusedxml==0.7.1 +starlette==0.47.2 +python-dotenv==1.1.1 +comm==0.2.3 +multidict==6.6.3 +python-json-logger==3.3.0 +jupyter_server==2.16.0 +nvidia-nvtx-cu12==12.6.77 +websockets==15.0.1 +referencing==0.36.2 +gguf==0.17.1 +lm-format-enforcer==0.10.12 +xformers==0.0.31 +ninja==1.11.1.4 +beautifulsoup4==4.13.4 +dnspython==2.7.0 +certifi==2025.8.3 +markdown-it-py==3.0.0 +stack-data==0.6.3 +nvidia-cusparselt-cu12==0.6.3 +networkx==3.5 +aiohappyeyeballs==2.6.1 +nvidia-cuda-cupti-cu12==12.6.80 +llguidance==0.7.30 +rich==14.1.0 +outlines_core==0.2.10 +nvidia-nccl-cu12==2.26.2 +pandas==2.3.1 +pydantic_core==2.33.2 +argon2-cffi==25.1.0 +babel==2.17.0 +isoduration==20.11.0 +uvloop==0.21.0 +json5==0.12.0 +widgetsnbextension==4.0.14 +aiohttp==3.12.15 +notebook_shim==0.2.4 +pandocfilters==1.5.1 +vllm==0.10.0 +blake3==1.0.5 +diskcache==5.6.3 +h11==0.16.0 +attrs==25.3.0 +nvidia-cusolver-cu12==11.7.1.2 +click==8.2.1 +cupy-cuda12x==13.5.1 +jedi==0.19.2 +py-cpuinfo==9.0.0 +overrides==7.7.0 +pytz==2025.2 +protobuf==6.31.1 +dill==0.3.8 +ipykernel==6.30.1 +frozenlist==1.7.0 +huggingface-hub==0.34.4 +rignore==0.6.4 +mistune==3.1.3 +sniffio==1.3.1 +httpx==0.28.1 +sentencepiece==0.2.0 +typing_extensions==4.14.1 +typing-inspection==0.4.1 +async-lru==2.0.5 +bioreason==0.1.0 +charset-normalizer==3.4.3 +llvmlite==0.44.0 +safetensors==0.6.2 +fqdn==1.5.1 +peft==0.17.0 +fastapi-cli==0.0.8 +torch==2.7.1 +jsonschema==4.25.0 +torchvision==0.22.1 +parso==0.8.4 +pillow==11.3.0 +pycountry==24.6.1 +scipy==1.16.1 +wandb==0.21.1 +pip==25.1 +wcwidth==0.2.13 +mistral_common==1.8.3 +nvidia-cudnn-cu12==9.5.1.17 +asttokens==3.0.0 +numpy==2.2.6 +uvicorn==0.35.0 +jupyter_client==8.6.3 +debugpy==1.8.16 +fastapi-cloud-cli==0.1.5 +compressed-tensors==0.10.2 +GitPython==3.1.45 +nvidia-ml-py==13.580.65 +tiktoken==0.11.0 +partial-json-parser==0.2.1.1.post6 +prompt_toolkit==3.0.51 +uri-template==1.3.0 +pyarrow==21.0.0 +nvidia-cuda-nvrtc-cu12==12.6.77 +propcache==0.3.2 +matplotlib-inline==0.1.7 +pure_eval==0.2.3 +nest-asyncio==1.6.0 +deepspeed==0.17.4 +einops==0.8.1 +triton==3.3.1 +trl==0.21.0 +pyzmq==27.0.1 +httptools==0.6.4 +ipython==9.4.0 +packaging==25.0 +hjson==3.1.0 +soxr==0.5.0.post1 +webcolors==24.11.1 +PyYAML==6.0.2 +ipywidgets==8.1.7 +lark==1.2.2 +python-multipart==0.0.20 +tinycss2==1.4.0 +mpmath==1.3.0 +psutil==7.0.0 +nbformat==5.10.4 +qwen-vl-utils==0.0.11 +notebook==7.4.5 +jupyterlab==4.4.5 +transformers==4.55.0 +urllib3==2.5.0 +terminado==0.18.1 +xgrammar==0.1.21 +xxhash==3.5.0 +tzdata==2025.2 +jupyterlab_widgets==3.0.15 +aiosignal==1.4.0 +annotated-types==0.7.0 +nvidia-nvjitlink-cu12==12.6.85 +httpcore==1.0.9 +pexpect==4.9.0 +tornado==6.5.2 +regex==2025.7.33 +filelock==3.18.0 +nvidia-cuda-runtime-cu12==12.6.77 +openai==1.90.0 +Pygments==2.19.2 +cffi==1.17.1 +nvidia-cufft-cu12==11.3.0.4 +rpds-py==0.27.0 +rich-toolkit==0.15.0 +bleach==6.2.0 +sympy==1.14.0 +python-dateutil==2.9.0.post0 +watchfiles==1.1.0 +nbclient==0.10.2 +hf-xet==1.1.7 +Send2Trash==1.8.3 +smmap==5.0.2 +tokenizers==0.21.4 +yarl==1.20.1 +MarkupSafe==3.0.2 +prometheus-fastapi-instrumentator==7.1.0 +rfc3986-validator==0.1.1 +bitsandbytes==0.46.1 +nvidia-cufile-cu12==1.11.1.6 +fsspec==2025.3.0 +executing==2.2.0 +pydantic==2.11.7 +jupyter_core==5.8.1 +msgspec==0.19.0 +nvidia-cusparse-cu12==12.5.4.2 +opencv-python-headless==4.12.0.88 +webencodings==0.5.1 +ptyprocess==0.7.0 +pytorch-lightning==2.5.2 +Jinja2==3.1.6 +websocket-client==1.8.0 +jsonpointer==3.0.0 +traitlets==5.14.3 +numba==0.61.2 +soupsieve==2.7 +cbor2==5.6.5 +multiprocess==0.70.16 +email_validator==2.2.0 +distro==1.9.0 +accelerate==1.10.0 +torchaudio==2.7.1 +jiter==0.10.0 +rfc3339-validator==0.1.4 +jupyter==1.1.1 +typer==0.16.0 +gitdb==4.0.12 +types-python-dateutil==2.9.0.20250809 +soundfile==0.13.1 +astor==0.8.1 +shellingham==1.5.4 +jupyterlab_server==2.27.3 +nvidia-curand-cu12==10.3.7.77 +jupyter-console==6.6.3 +msgpack==1.1.1 +six==1.17.0 +pycparser==2.22 +arrow==1.3.0 +tqdm==4.67.1 +lightning-utilities==0.15.2 +interegular==0.3.3 +rfc3987-syntax==1.1.0 +ipython_pygments_lexers==1.1.1 +datasets==4.0.0 +jupyter_server_terminals==0.5.3 +platformdirs==4.3.8 +pydantic-extra-types==2.10.5 +decorator==5.2.1 +mdurl==0.1.2 +nvidia-cublas-cu12==12.6.4.1 +jsonschema-specifications==2025.4.1 +setuptools==78.1.1 +fastrlock==0.8.3 +argon2-cffi-bindings==25.1.0 +nbconvert==7.16.6 diff --git a/BioReason_new/wandb/run-20250813_121906-3wcb4wer/files/wandb-metadata.json b/BioReason_new/wandb/run-20250813_121906-3wcb4wer/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..a436367d4bbe97ec44dbeb989269c7ad817a4579 --- /dev/null +++ b/BioReason_new/wandb/run-20250813_121906-3wcb4wer/files/wandb-metadata.json @@ -0,0 +1,116 @@ +{ + "os": "Linux-5.10.134-008.18.kangaroo.al8.x86_64-x86_64-with-glibc2.35", + "python": "CPython 3.11.0", + "startedAt": "2025-08-13T04:19:06.448075Z", + "args": [ + "--text_model_name", + "/oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged", + "--protein_model_name", + "/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m", + "--qformer_model_name", + "/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft", + "--num_query_tokens", + "8", + "--train_dataset", + "/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl", + "--valid_dataset", + "/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl", + "--output_dir", + "./contrastive_outputs", + "--num_epochs", + "10", + "--batch_size", + "32", + "--learning_rate", + "1e-4", + "--temperature", + "0.07", + "--freeze_protein_model", + "--freeze_text_model", + "--enable_ptm", + "--max_length_protein", + "1024", + "--max_length_text", + "512", + "--num_workers", + "8", + "--eval_dataset", + "--use_wandb", + "--wandb_project", + "protein-llm-contrastive", + "--logging_steps", + "100", + "--eval_steps", + "500", + "--save_steps", + "1000" + ], + "program": "/nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py", + "codePath": "wangyujia/BioReason_new/train_contrastive.py", + "codePathLocal": "train_contrastive.py", + "git": { + "remote": "https://github.com/PorUna-byte/PAR.git", + "commit": "b8caf406aa1699c788f0ca6e44a1769452c317db" + }, + "email": "gia0603yucca@gmail.com", + "root": "/nas/shared/kilab/wangyujia/BioReason_new", + "host": "dsw-265304-585cc9d768-ckfd7", + "executable": "/root/miniconda3/envs/bioreason/bin/python", + "cpu_count": 64, + "cpu_count_logical": 64, + "gpu": "NVIDIA A800-SXM4-80GB", + "gpu_count": 8, + "disk": { + "/": { + "total": "1623302262784", + "used": "1014808576" + } + }, + "memory": { + "total": "549755813888" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-62fcc532-880f-7324-eb13-3b739a38c29b" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-af1197aa-3e7d-a53e-185d-04b1d64164da" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-3e6e551f-2d7d-a27b-654a-da56933d2283" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-a947b64a-c545-ee5e-30e6-b4b0e2dd8fdc" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-ebd04f3f-c437-1b8b-26f2-4b1ebfee93c4" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-53994891-72a6-25be-a4c8-e32c695155b0" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-a927d00a-15c5-31f5-9ae1-9d2e773821a1" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-93d555c4-29a0-cd78-abac-d98cd160d4e1" + } + ], + "cudaVersion": "12.1", + "writerId": "eku61gd0lj70gwdugjle81nramcib5o4" +} \ No newline at end of file diff --git a/BioReason_new/wandb/run-20250813_121906-3wcb4wer/files/wandb-summary.json b/BioReason_new/wandb/run-20250813_121906-3wcb4wer/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..10d12aeda4acc1438b9fd2d4a6fe423737b697cb --- /dev/null +++ b/BioReason_new/wandb/run-20250813_121906-3wcb4wer/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":1910},"_runtime":1910} \ No newline at end of file diff --git a/BioReason_new/wandb/run-20250813_121906-3wcb4wer/logs/debug-internal.log b/BioReason_new/wandb/run-20250813_121906-3wcb4wer/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..70be736d6bee1d75045033bc562f2758ffe761a4 --- /dev/null +++ b/BioReason_new/wandb/run-20250813_121906-3wcb4wer/logs/debug-internal.log @@ -0,0 +1,18 @@ +{"time":"2025-08-13T12:19:06.822910203+08:00","level":"INFO","msg":"stream: starting","core version":"0.21.1"} +{"time":"2025-08-13T12:19:08.068655385+08:00","level":"INFO","msg":"stream: created new stream","id":"3wcb4wer"} +{"time":"2025-08-13T12:19:08.068708947+08:00","level":"INFO","msg":"stream: started","id":"3wcb4wer"} +{"time":"2025-08-13T12:19:08.068747833+08:00","level":"INFO","msg":"writer: started","stream_id":"3wcb4wer"} +{"time":"2025-08-13T12:19:08.068766174+08:00","level":"INFO","msg":"sender: started","stream_id":"3wcb4wer"} +{"time":"2025-08-13T12:19:08.06880027+08:00","level":"INFO","msg":"handler: started","stream_id":"3wcb4wer"} +{"time":"2025-08-13T12:19:09.937699234+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/3wcb4wer/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T041909Z&X-Goog-Expires=86399&X-Goog-Signature=2235d1f23ffa94e46560d8d9f1d99a989e9a6742f86a4808f9aeb398930bff09a6648b165f02c86e3336b2c45dea184c1c00bfc510a911f21d1d3c93af8315df007c4be4d48a40d509eddf912bad9dad6111e122f0c52e4718f52e9920b371ccf7d661507efed6a4dca98d6905e4aa52888a2f967d3b5cb0a7edb86ae58b67aa789d6b6d0baeadacdb94ddddd35daa1abcc27411f73ce268720d2a07f37be8e791e57cb72538e70626bd1fa9f490e39a22eeeb4e0ba62fa31b4209ee026d51d1e750551a6dcbfd81ee69fd25b7a1377e4688b46f170feafeb3a758aa1d9e1b881ee4009e8798452e7af98f262897f880de73f172a6426c917978ef0925e2e600&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:51032->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/3wcb4wer/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T041909Z&X-Goog-Expires=86399&X-Goog-Signature=2235d1f23ffa94e46560d8d9f1d99a989e9a6742f86a4808f9aeb398930bff09a6648b165f02c86e3336b2c45dea184c1c00bfc510a911f21d1d3c93af8315df007c4be4d48a40d509eddf912bad9dad6111e122f0c52e4718f52e9920b371ccf7d661507efed6a4dca98d6905e4aa52888a2f967d3b5cb0a7edb86ae58b67aa789d6b6d0baeadacdb94ddddd35daa1abcc27411f73ce268720d2a07f37be8e791e57cb72538e70626bd1fa9f490e39a22eeeb4e0ba62fa31b4209ee026d51d1e750551a6dcbfd81ee69fd25b7a1377e4688b46f170feafeb3a758aa1d9e1b881ee4009e8798452e7af98f262897f880de73f172a6426c917978ef0925e2e600&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T12:21:20.029385617+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/3wcb4wer/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T041909Z&X-Goog-Expires=86399&X-Goog-Signature=810610a47880236a95ccb7704d4c0dc16f7a0b7eae65667872dce5f537b288a571822f41e7a45f3e344f5d667ba7e46b08bfe38190c6996e05cc6271d589340f91b659b04ad279a66aaf29fc5c234107aabf0c722489b48ea64d9be719128f32bc5d56b7b2f19e2bb2e0134714913c7279b194400e5b22f24cbaa2b825742be756fbb257747c6677c2e141f437e9bb64d7ff4c573fde01df4f40d20a5ba4f23853dc33d89cdfc237721a16212ba8ad2d8d30430713ee477b5187b4ccb50edd3683597d5a99258908fe9e74178a8be41e73ab77fe17889dcd7542db1f8ceaa8c5fe918adf111dfdd605f40c19e93f9ad543d872111ec416aa3cb5e60909c01345&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:36830->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/3wcb4wer/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T041909Z&X-Goog-Expires=86399&X-Goog-Signature=810610a47880236a95ccb7704d4c0dc16f7a0b7eae65667872dce5f537b288a571822f41e7a45f3e344f5d667ba7e46b08bfe38190c6996e05cc6271d589340f91b659b04ad279a66aaf29fc5c234107aabf0c722489b48ea64d9be719128f32bc5d56b7b2f19e2bb2e0134714913c7279b194400e5b22f24cbaa2b825742be756fbb257747c6677c2e141f437e9bb64d7ff4c573fde01df4f40d20a5ba4f23853dc33d89cdfc237721a16212ba8ad2d8d30430713ee477b5187b4ccb50edd3683597d5a99258908fe9e74178a8be41e73ab77fe17889dcd7542db1f8ceaa8c5fe918adf111dfdd605f40c19e93f9ad543d872111ec416aa3cb5e60909c01345&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T12:21:22.635640911+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/3wcb4wer/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T041909Z&X-Goog-Expires=86399&X-Goog-Signature=810610a47880236a95ccb7704d4c0dc16f7a0b7eae65667872dce5f537b288a571822f41e7a45f3e344f5d667ba7e46b08bfe38190c6996e05cc6271d589340f91b659b04ad279a66aaf29fc5c234107aabf0c722489b48ea64d9be719128f32bc5d56b7b2f19e2bb2e0134714913c7279b194400e5b22f24cbaa2b825742be756fbb257747c6677c2e141f437e9bb64d7ff4c573fde01df4f40d20a5ba4f23853dc33d89cdfc237721a16212ba8ad2d8d30430713ee477b5187b4ccb50edd3683597d5a99258908fe9e74178a8be41e73ab77fe17889dcd7542db1f8ceaa8c5fe918adf111dfdd605f40c19e93f9ad543d872111ec416aa3cb5e60909c01345&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:36836->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/3wcb4wer/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T041909Z&X-Goog-Expires=86399&X-Goog-Signature=810610a47880236a95ccb7704d4c0dc16f7a0b7eae65667872dce5f537b288a571822f41e7a45f3e344f5d667ba7e46b08bfe38190c6996e05cc6271d589340f91b659b04ad279a66aaf29fc5c234107aabf0c722489b48ea64d9be719128f32bc5d56b7b2f19e2bb2e0134714913c7279b194400e5b22f24cbaa2b825742be756fbb257747c6677c2e141f437e9bb64d7ff4c573fde01df4f40d20a5ba4f23853dc33d89cdfc237721a16212ba8ad2d8d30430713ee477b5187b4ccb50edd3683597d5a99258908fe9e74178a8be41e73ab77fe17889dcd7542db1f8ceaa8c5fe918adf111dfdd605f40c19e93f9ad543d872111ec416aa3cb5e60909c01345&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T12:21:28.126891685+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/3wcb4wer/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T041909Z&X-Goog-Expires=86399&X-Goog-Signature=810610a47880236a95ccb7704d4c0dc16f7a0b7eae65667872dce5f537b288a571822f41e7a45f3e344f5d667ba7e46b08bfe38190c6996e05cc6271d589340f91b659b04ad279a66aaf29fc5c234107aabf0c722489b48ea64d9be719128f32bc5d56b7b2f19e2bb2e0134714913c7279b194400e5b22f24cbaa2b825742be756fbb257747c6677c2e141f437e9bb64d7ff4c573fde01df4f40d20a5ba4f23853dc33d89cdfc237721a16212ba8ad2d8d30430713ee477b5187b4ccb50edd3683597d5a99258908fe9e74178a8be41e73ab77fe17889dcd7542db1f8ceaa8c5fe918adf111dfdd605f40c19e93f9ad543d872111ec416aa3cb5e60909c01345&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:49190->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/3wcb4wer/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T041909Z&X-Goog-Expires=86399&X-Goog-Signature=810610a47880236a95ccb7704d4c0dc16f7a0b7eae65667872dce5f537b288a571822f41e7a45f3e344f5d667ba7e46b08bfe38190c6996e05cc6271d589340f91b659b04ad279a66aaf29fc5c234107aabf0c722489b48ea64d9be719128f32bc5d56b7b2f19e2bb2e0134714913c7279b194400e5b22f24cbaa2b825742be756fbb257747c6677c2e141f437e9bb64d7ff4c573fde01df4f40d20a5ba4f23853dc33d89cdfc237721a16212ba8ad2d8d30430713ee477b5187b4ccb50edd3683597d5a99258908fe9e74178a8be41e73ab77fe17889dcd7542db1f8ceaa8c5fe918adf111dfdd605f40c19e93f9ad543d872111ec416aa3cb5e60909c01345&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T12:23:35.200988988+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/3wcb4wer/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T041909Z&X-Goog-Expires=86399&X-Goog-Signature=2235d1f23ffa94e46560d8d9f1d99a989e9a6742f86a4808f9aeb398930bff09a6648b165f02c86e3336b2c45dea184c1c00bfc510a911f21d1d3c93af8315df007c4be4d48a40d509eddf912bad9dad6111e122f0c52e4718f52e9920b371ccf7d661507efed6a4dca98d6905e4aa52888a2f967d3b5cb0a7edb86ae58b67aa789d6b6d0baeadacdb94ddddd35daa1abcc27411f73ce268720d2a07f37be8e791e57cb72538e70626bd1fa9f490e39a22eeeb4e0ba62fa31b4209ee026d51d1e750551a6dcbfd81ee69fd25b7a1377e4688b46f170feafeb3a758aa1d9e1b881ee4009e8798452e7af98f262897f880de73f172a6426c917978ef0925e2e600&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:59810->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/3wcb4wer/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T041909Z&X-Goog-Expires=86399&X-Goog-Signature=2235d1f23ffa94e46560d8d9f1d99a989e9a6742f86a4808f9aeb398930bff09a6648b165f02c86e3336b2c45dea184c1c00bfc510a911f21d1d3c93af8315df007c4be4d48a40d509eddf912bad9dad6111e122f0c52e4718f52e9920b371ccf7d661507efed6a4dca98d6905e4aa52888a2f967d3b5cb0a7edb86ae58b67aa789d6b6d0baeadacdb94ddddd35daa1abcc27411f73ce268720d2a07f37be8e791e57cb72538e70626bd1fa9f490e39a22eeeb4e0ba62fa31b4209ee026d51d1e750551a6dcbfd81ee69fd25b7a1377e4688b46f170feafeb3a758aa1d9e1b881ee4009e8798452e7af98f262897f880de73f172a6426c917978ef0925e2e600&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T12:32:39.216496055+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": read tcp 10.1.1.90:51308->104.21.20.172:443: read: connection reset by peer"} +{"time":"2025-08-13T12:50:59.026820434+08:00","level":"INFO","msg":"stream: closing","id":"3wcb4wer"} +{"time":"2025-08-13T12:51:02.081328021+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/3wcb4wer/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T045059Z&X-Goog-Expires=86399&X-Goog-Signature=8d0f986d028f03059f3bff491d651565b582576cfdafdfdab6e66ea093da1343802941babac635335ed038e1bbd1238c743082aa2ed1185361180b4f0975e9e8561cd1ac3d40566f176e68fe8dda7d344fdef70b0e12d916df0b76e86443f651201dead7fa09deea52c190864f6ecce2d3e92a995b757b5e0ad59abf85d627116927c6c77b3f1cd267c1e1067f3d75c679404c31711964312ef6c8d5b2b03f7feae1450de7e3ac7842fbde41e5c5cffd757e2e5d8e8ecd410bf9b6256bd6d39d26c3045758a8d88ebb2250e9a4cfe3b7acdcf7c3b56cdf9bf7ac63e04aa186a332dc0859f1c10ee38d9bf18eaa589d01c332f04a428612ef98ade0a0dba09257&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:44048->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/3wcb4wer/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T045059Z&X-Goog-Expires=86399&X-Goog-Signature=8d0f986d028f03059f3bff491d651565b582576cfdafdfdab6e66ea093da1343802941babac635335ed038e1bbd1238c743082aa2ed1185361180b4f0975e9e8561cd1ac3d40566f176e68fe8dda7d344fdef70b0e12d916df0b76e86443f651201dead7fa09deea52c190864f6ecce2d3e92a995b757b5e0ad59abf85d627116927c6c77b3f1cd267c1e1067f3d75c679404c31711964312ef6c8d5b2b03f7feae1450de7e3ac7842fbde41e5c5cffd757e2e5d8e8ecd410bf9b6256bd6d39d26c3045758a8d88ebb2250e9a4cfe3b7acdcf7c3b56cdf9bf7ac63e04aa186a332dc0859f1c10ee38d9bf18eaa589d01c332f04a428612ef98ade0a0dba09257&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T12:51:05.123245787+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-08-13T12:51:09.665755774+08:00","level":"INFO","msg":"handler: closed","stream_id":"3wcb4wer"} +{"time":"2025-08-13T12:51:09.669627695+08:00","level":"INFO","msg":"sender: closed","stream_id":"3wcb4wer"} +{"time":"2025-08-13T12:51:09.669650417+08:00","level":"INFO","msg":"stream: closed","id":"3wcb4wer"} diff --git a/BioReason_new/wandb/run-20250813_121906-3wcb4wer/logs/debug.log b/BioReason_new/wandb/run-20250813_121906-3wcb4wer/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..3932841a24c7aaeaa49f62588ea223aa26a11318 --- /dev/null +++ b/BioReason_new/wandb/run-20250813_121906-3wcb4wer/logs/debug.log @@ -0,0 +1,23 @@ +2025-08-13 12:19:06,598 INFO MainThread:2035 [wandb_setup.py:_flush():80] Current SDK version is 0.21.1 +2025-08-13 12:19:06,598 INFO MainThread:2035 [wandb_setup.py:_flush():80] Configure stats pid to 2035 +2025-08-13 12:19:06,598 INFO MainThread:2035 [wandb_setup.py:_flush():80] Loading settings from /root/.config/wandb/settings +2025-08-13 12:19:06,598 INFO MainThread:2035 [wandb_setup.py:_flush():80] Loading settings from /nas/shared/kilab/wangyujia/BioReason_new/wandb/settings +2025-08-13 12:19:06,598 INFO MainThread:2035 [wandb_setup.py:_flush():80] Loading settings from environment variables +2025-08-13 12:19:06,598 INFO MainThread:2035 [wandb_init.py:setup_run_log_directory():703] Logging user logs to /nas/shared/kilab/wangyujia/BioReason_new/wandb/run-20250813_121906-3wcb4wer/logs/debug.log +2025-08-13 12:19:06,598 INFO MainThread:2035 [wandb_init.py:setup_run_log_directory():704] Logging internal logs to /nas/shared/kilab/wangyujia/BioReason_new/wandb/run-20250813_121906-3wcb4wer/logs/debug-internal.log +2025-08-13 12:19:06,599 INFO MainThread:2035 [wandb_init.py:init():830] calling init triggers +2025-08-13 12:19:06,599 INFO MainThread:2035 [wandb_init.py:init():835] wandb.init called with sweep_config: {} +config: {'text_model_name': '/oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged', 'protein_model_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m', 'qformer_model_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft', 'cache_dir': '/model-weights', 'num_query_tokens': 8, 'train_dataset': '/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl', 'valid_dataset': '/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl', 'eval_dataset': True, 'output_dir': './contrastive_outputs', 'num_epochs': 10, 'batch_size': 32, 'learning_rate': 0.0001, 'weight_decay': 0.01, 'warmup_steps': 1000, 'gradient_accumulation_steps': 1, 'temperature': 0.07, 'freeze_protein_model': True, 'freeze_text_model': True, 'protein_weight': 1.0, 'text_weight': 1.0, 'enable_ptm': True, 'ptm_weight': 1.0, 'max_length_protein': 1024, 'max_length_text': 512, 'num_workers': 8, 'logging_steps': 100, 'eval_steps': 500, 'save_steps': 1000, 'save_total_limit': 3, 'fp16': False, 'bf16': False, 'seed': 42, 'use_wandb': True, 'wandb_project': 'protein-llm-contrastive', 'wandb_entity': None, '_wandb': {}} +2025-08-13 12:19:06,599 INFO MainThread:2035 [wandb_init.py:init():871] starting backend +2025-08-13 12:19:06,809 INFO MainThread:2035 [wandb_init.py:init():874] sending inform_init request +2025-08-13 12:19:06,816 INFO MainThread:2035 [wandb_init.py:init():882] backend started and connected +2025-08-13 12:19:06,819 INFO MainThread:2035 [wandb_init.py:init():953] updated telemetry +2025-08-13 12:19:06,844 INFO MainThread:2035 [wandb_init.py:init():977] communicating run to backend with 90.0 second timeout +2025-08-13 12:19:08,807 INFO MainThread:2035 [wandb_init.py:init():1029] starting run threads in backend +2025-08-13 12:19:08,949 INFO MainThread:2035 [wandb_run.py:_console_start():2494] atexit reg +2025-08-13 12:19:08,949 INFO MainThread:2035 [wandb_run.py:_redirect():2342] redirect: wrap_raw +2025-08-13 12:19:08,949 INFO MainThread:2035 [wandb_run.py:_redirect():2411] Wrapping output streams. +2025-08-13 12:19:08,949 INFO MainThread:2035 [wandb_run.py:_redirect():2434] Redirects installed. +2025-08-13 12:19:08,952 INFO MainThread:2035 [wandb_init.py:init():1075] run started, returning control to user process +2025-08-13 12:50:52,264 INFO MainThread:2035 [wandb_run.py:_config_callback():1380] config_cb None None {'output_dir': './contrastive_outputs', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': True, 'do_predict': False, 'eval_strategy': 'steps', 'prediction_loss_only': False, 'per_device_train_batch_size': 32, 'per_device_eval_batch_size': 32, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'eval_delay': 0, 'torch_empty_cache_steps': None, 'learning_rate': 0.0001, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 10, 'max_steps': -1, 'lr_scheduler_type': 'linear', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.0, 'warmup_steps': 1000, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': './contrastive_outputs/runs/Aug13_12-50-37_dsw-265304-585cc9d768-ckfd7', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 100, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 1000, 'save_total_limit': 3, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'restore_callback_states_from_checkpoint': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 42, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': False, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 500, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': None, 'disable_tqdm': False, 'remove_unused_columns': False, 'label_names': None, 'load_best_model_at_end': True, 'metric_for_best_model': 'eval_avg_recall_at_1', 'greater_is_better': True, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'deepspeed': None, 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['wandb'], 'ddp_find_unused_parameters': False, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '', 'hub_private_repo': None, 'hub_always_push': False, 'hub_revision': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'include_for_metrics': [], 'eval_do_concat_batches': True, 'fp16_backend': 'auto', 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None, 'batch_eval_metrics': False, 'eval_on_start': False, 'use_liger_kernel': False, 'liger_kernel_config': None, 'eval_use_gather_object': False, 'average_tokens_across_devices': False, 'temperature': 0.07, 'freeze_protein_model': True, 'freeze_text_model': True, 'protein_weight': 1.0, 'text_weight': 1.0, 'max_length_protein': 1024, 'max_length_text': 512, 'enable_ptm': True, 'ptm_weight': 1.0} +2025-08-13 12:50:59,025 INFO MsgRouterThr:2035 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles. diff --git a/BioReason_new/wandb/run-20250813_125236-re7fb96i/files/config.yaml b/BioReason_new/wandb/run-20250813_125236-re7fb96i/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8def581be5daf66504c53ae7c7d999349c380680 --- /dev/null +++ b/BioReason_new/wandb/run-20250813_125236-re7fb96i/files/config.yaml @@ -0,0 +1,458 @@ +_wandb: + value: + cli_version: 0.21.1 + e: + gef9d42v3ap5mwuxow19xbx613zovmhi: + args: + - --text_model_name + - /oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged + - --protein_model_name + - /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m + - --qformer_model_name + - /nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft + - --num_query_tokens + - "8" + - --train_dataset + - /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl + - --valid_dataset + - /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl + - --output_dir + - ./contrastive_outputs + - --num_epochs + - "10" + - --batch_size + - "8" + - --learning_rate + - "1e-4" + - --temperature + - "0.07" + - --freeze_protein_model + - --freeze_text_model + - --enable_ptm + - --max_length_protein + - "1024" + - --max_length_text + - "512" + - --num_workers + - "8" + - --eval_dataset + - --use_wandb + - --wandb_project + - protein-llm-contrastive + - --logging_steps + - "100" + - --eval_steps + - "500" + - --save_steps + - "1000" + codePath: wangyujia/BioReason_new/train_contrastive.py + codePathLocal: train_contrastive.py + cpu_count: 64 + cpu_count_logical: 64 + cudaVersion: "12.1" + disk: + /: + total: "1623302262784" + used: "1015009280" + email: gia0603yucca@gmail.com + executable: /root/miniconda3/envs/bioreason/bin/python + git: + commit: b8caf406aa1699c788f0ca6e44a1769452c317db + remote: https://github.com/PorUna-byte/PAR.git + gpu: NVIDIA A800-SXM4-80GB + gpu_count: 8 + gpu_nvidia: + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-62fcc532-880f-7324-eb13-3b739a38c29b + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-af1197aa-3e7d-a53e-185d-04b1d64164da + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-3e6e551f-2d7d-a27b-654a-da56933d2283 + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-a947b64a-c545-ee5e-30e6-b4b0e2dd8fdc + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-ebd04f3f-c437-1b8b-26f2-4b1ebfee93c4 + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-53994891-72a6-25be-a4c8-e32c695155b0 + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-a927d00a-15c5-31f5-9ae1-9d2e773821a1 + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-93d555c4-29a0-cd78-abac-d98cd160d4e1 + host: dsw-265304-585cc9d768-ckfd7 + memory: + total: "549755813888" + os: Linux-5.10.134-008.18.kangaroo.al8.x86_64-x86_64-with-glibc2.35 + program: /nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py + python: CPython 3.11.0 + root: /nas/shared/kilab/wangyujia/BioReason_new + startedAt: "2025-08-13T04:52:36.303807Z" + writerId: gef9d42v3ap5mwuxow19xbx613zovmhi + m: + - "1": train/global_step + "6": + - 3 + "7": [] + - "2": '*' + "5": 1 + "6": + - 1 + "7": [] + python_version: 3.11.0 + t: + "1": + - 1 + - 9 + - 11 + - 41 + - 49 + - 51 + - 71 + - 84 + - 98 + - 103 + "2": + - 1 + - 9 + - 11 + - 41 + - 49 + - 51 + - 71 + - 84 + - 98 + - 103 + "3": + - 7 + - 13 + - 16 + - 66 + "4": 3.11.0 + "5": 0.21.1 + "6": 4.55.0 + "9": + "1": transformers_trainer + "12": 0.21.1 + "13": linux-x86_64 +accelerator_config: + value: + dispatch_batches: null + even_batches: true + gradient_accumulation_kwargs: null + non_blocking: false + split_batches: false + use_seedable_sampler: true +adafactor: + value: false +adam_beta1: + value: 0.9 +adam_beta2: + value: 0.999 +adam_epsilon: + value: 1e-08 +auto_find_batch_size: + value: false +average_tokens_across_devices: + value: false +batch_eval_metrics: + value: false +batch_size: + value: 8 +bf16: + value: false +bf16_full_eval: + value: false +cache_dir: + value: /model-weights +data_seed: + value: null +dataloader_drop_last: + value: false +dataloader_num_workers: + value: 8 +dataloader_persistent_workers: + value: false +dataloader_pin_memory: + value: true +dataloader_prefetch_factor: + value: null +ddp_backend: + value: null +ddp_broadcast_buffers: + value: null +ddp_bucket_cap_mb: + value: null +ddp_find_unused_parameters: + value: false +ddp_timeout: + value: 1800 +debug: + value: [] +deepspeed: + value: null +disable_tqdm: + value: false +do_eval: + value: true +do_predict: + value: false +do_train: + value: false +enable_ptm: + value: true +eval_accumulation_steps: + value: null +eval_dataset: + value: true +eval_delay: + value: 0 +eval_do_concat_batches: + value: true +eval_on_start: + value: false +eval_steps: + value: 500 +eval_strategy: + value: steps +eval_use_gather_object: + value: false +fp16: + value: false +fp16_backend: + value: auto +fp16_full_eval: + value: false +fp16_opt_level: + value: O1 +freeze_protein_model: + value: true +freeze_text_model: + value: true +fsdp: + value: [] +fsdp_config: + value: + min_num_params: 0 + xla: false + xla_fsdp_grad_ckpt: false + xla_fsdp_v2: false +fsdp_min_num_params: + value: 0 +fsdp_transformer_layer_cls_to_wrap: + value: null +full_determinism: + value: false +gradient_accumulation_steps: + value: 1 +gradient_checkpointing: + value: false +gradient_checkpointing_kwargs: + value: null +greater_is_better: + value: true +group_by_length: + value: false +half_precision_backend: + value: auto +hub_always_push: + value: false +hub_model_id: + value: null +hub_private_repo: + value: null +hub_revision: + value: null +hub_strategy: + value: every_save +hub_token: + value: +ignore_data_skip: + value: false +include_for_metrics: + value: [] +include_inputs_for_metrics: + value: false +include_num_input_tokens_seen: + value: false +include_tokens_per_second: + value: false +jit_mode_eval: + value: false +label_names: + value: null +label_smoothing_factor: + value: 0 +learning_rate: + value: 0.0001 +length_column_name: + value: length +liger_kernel_config: + value: null +load_best_model_at_end: + value: true +local_rank: + value: 0 +log_level: + value: passive +log_level_replica: + value: warning +log_on_each_node: + value: true +logging_dir: + value: ./contrastive_outputs/runs/Aug13_12-52-45_dsw-265304-585cc9d768-ckfd7 +logging_first_step: + value: false +logging_nan_inf_filter: + value: true +logging_steps: + value: 100 +logging_strategy: + value: steps +lr_scheduler_type: + value: linear +max_grad_norm: + value: 1 +max_length_protein: + value: 1024 +max_length_text: + value: 512 +max_steps: + value: -1 +metric_for_best_model: + value: eval_avg_recall_at_1 +mp_parameters: + value: "" +neftune_noise_alpha: + value: null +no_cuda: + value: false +num_epochs: + value: 10 +num_query_tokens: + value: 8 +num_train_epochs: + value: 10 +num_workers: + value: 8 +optim: + value: adamw_torch +optim_args: + value: null +optim_target_modules: + value: null +output_dir: + value: ./contrastive_outputs +overwrite_output_dir: + value: false +past_index: + value: -1 +per_device_eval_batch_size: + value: 8 +per_device_train_batch_size: + value: 8 +per_gpu_eval_batch_size: + value: null +per_gpu_train_batch_size: + value: null +prediction_loss_only: + value: false +protein_model_name: + value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m +protein_weight: + value: 1 +ptm_weight: + value: 1 +push_to_hub: + value: false +push_to_hub_model_id: + value: null +push_to_hub_organization: + value: null +push_to_hub_token: + value: +qformer_model_name: + value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft +ray_scope: + value: last +remove_unused_columns: + value: false +report_to: + value: + - wandb +restore_callback_states_from_checkpoint: + value: false +resume_from_checkpoint: + value: null +run_name: + value: null +save_on_each_node: + value: false +save_only_model: + value: false +save_safetensors: + value: true +save_steps: + value: 1000 +save_strategy: + value: steps +save_total_limit: + value: 3 +seed: + value: 42 +skip_memory_metrics: + value: true +temperature: + value: 0.07 +text_model_name: + value: /oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged +text_weight: + value: 1 +tf32: + value: null +torch_compile: + value: false +torch_compile_backend: + value: null +torch_compile_mode: + value: null +torch_empty_cache_steps: + value: null +torchdynamo: + value: null +tpu_metrics_debug: + value: false +tpu_num_cores: + value: null +train_dataset: + value: /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl +use_cpu: + value: false +use_ipex: + value: false +use_legacy_prediction_loop: + value: false +use_liger_kernel: + value: false +use_mps_device: + value: false +use_wandb: + value: true +valid_dataset: + value: /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl +wandb_entity: + value: null +wandb_project: + value: protein-llm-contrastive +warmup_ratio: + value: 0 +warmup_steps: + value: 1000 +weight_decay: + value: 0.01 diff --git a/BioReason_new/wandb/run-20250813_125236-re7fb96i/files/output.log b/BioReason_new/wandb/run-20250813_125236-re7fb96i/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..f635bc7703af1e22be22fa98862e9098cb6c7d65 --- /dev/null +++ b/BioReason_new/wandb/run-20250813_125236-re7fb96i/files/output.log @@ -0,0 +1,57 @@ +Loading model... +Loading checkpoint shards: 100%|██████████████████████████████████████████████████████| 4/4 [00:01<00:00, 2.43it/s] +Some weights of EsmModel were not initialized from the model checkpoint at /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m and are newly initialized: ['pooler.dense.bias', 'pooler.dense.weight'] +You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference. +Loading datasets... +Training dataset size: 430595 +Eval dataset size: 10000 +Initializing enhanced trainer... +[2025-08-13 12:52:50,263] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) +/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/_distutils_hack/__init__.py:53: UserWarning: Reliance on distutils from stdlib is deprecated. Users must rely on setuptools to provide the distutils module. Avoid importing distutils or import setuptools first, and avoid setting SETUPTOOLS_USE_DISTUTILS=stdlib. Register concerns at https://github.com/pypa/setuptools/issues/new?template=distutils-deprecation.yml + warnings.warn( +[2025-08-13 12:52:51,304] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False +Starting enhanced contrastive training... +- Contrastive learning enabled +- Protein-text matching: enabled +- Temperature: 0.07 +- PTM weight: 1.0 + 0%| | 0/67290 [00:00 + trainer = main(args) + ^^^^^^^^^^ + File "/nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py", line 399, in main + trainer.train() + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/trainer.py", line 2238, in train + return inner_training_loop( + ^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/trainer.py", line 2582, in _inner_training_loop + tr_loss_step = self.training_step(model, inputs, num_items_in_batch) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/trainer.py", line 3796, in training_step + loss = self.compute_loss(model, inputs, num_items_in_batch=num_items_in_batch) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/nas/shared/kilab/wangyujia/BioReason_new/bioreason/trainer/contrast_trainer_new.py", line 289, in compute_loss + protein_features = model.get_protein_features(protein_sequences) # [B, num_queries, embed_dim] + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/nas/shared/kilab/wangyujia/BioReason_new/bioreason/models/protein_llm.py", line 429, in get_protein_features + protein_features = self.protein_projection( + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1762, in _call_impl + return forward_call(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/nas/shared/kilab/wangyujia/BioReason_new/bioreason/models/protein_llm.py", line 98, in forward + outputs = self.qformer( + ^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1762, in _call_impl + return forward_call(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/models/bert/modeling_bert.py", line 937, in forward + buffered_token_type_ids_expanded = buffered_token_type_ids.expand(batch_size, seq_length) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +RuntimeError: The expanded size of the tensor (967) must match the existing size (512) at non-singleton dimension 1. Target sizes: [64, 967]. Tensor sizes: [1, 512] diff --git a/BioReason_new/wandb/run-20250813_125236-re7fb96i/files/requirements.txt b/BioReason_new/wandb/run-20250813_125236-re7fb96i/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..743d03fe27e0b7e200935023c07a1d7fa61b99ce --- /dev/null +++ b/BioReason_new/wandb/run-20250813_125236-re7fb96i/files/requirements.txt @@ -0,0 +1,233 @@ +jupyterlab_pygments==0.3.0 +depyf==0.19.0 +av==15.0.0 +fastapi==0.116.1 +ray==2.48.0 +cachetools==6.1.0 +prometheus_client==0.22.1 +requests==2.32.4 +jupyter-lsp==2.2.6 +jupyter-events==0.12.0 +anyio==4.10.0 +pybase64==1.4.2 +fastjsonschema==2.21.1 +cloudpickle==3.1.1 +sentry-sdk==2.34.1 +torchmetrics==1.8.1 +wheel==0.45.1 +idna==3.10 +defusedxml==0.7.1 +starlette==0.47.2 +python-dotenv==1.1.1 +comm==0.2.3 +multidict==6.6.3 +python-json-logger==3.3.0 +jupyter_server==2.16.0 +nvidia-nvtx-cu12==12.6.77 +websockets==15.0.1 +referencing==0.36.2 +gguf==0.17.1 +lm-format-enforcer==0.10.12 +xformers==0.0.31 +ninja==1.11.1.4 +beautifulsoup4==4.13.4 +dnspython==2.7.0 +certifi==2025.8.3 +markdown-it-py==3.0.0 +stack-data==0.6.3 +nvidia-cusparselt-cu12==0.6.3 +networkx==3.5 +aiohappyeyeballs==2.6.1 +nvidia-cuda-cupti-cu12==12.6.80 +llguidance==0.7.30 +rich==14.1.0 +outlines_core==0.2.10 +nvidia-nccl-cu12==2.26.2 +pandas==2.3.1 +pydantic_core==2.33.2 +argon2-cffi==25.1.0 +babel==2.17.0 +isoduration==20.11.0 +uvloop==0.21.0 +json5==0.12.0 +widgetsnbextension==4.0.14 +aiohttp==3.12.15 +notebook_shim==0.2.4 +pandocfilters==1.5.1 +vllm==0.10.0 +blake3==1.0.5 +diskcache==5.6.3 +h11==0.16.0 +attrs==25.3.0 +nvidia-cusolver-cu12==11.7.1.2 +click==8.2.1 +cupy-cuda12x==13.5.1 +jedi==0.19.2 +py-cpuinfo==9.0.0 +overrides==7.7.0 +pytz==2025.2 +protobuf==6.31.1 +dill==0.3.8 +ipykernel==6.30.1 +frozenlist==1.7.0 +huggingface-hub==0.34.4 +rignore==0.6.4 +mistune==3.1.3 +sniffio==1.3.1 +httpx==0.28.1 +sentencepiece==0.2.0 +typing_extensions==4.14.1 +typing-inspection==0.4.1 +async-lru==2.0.5 +bioreason==0.1.0 +charset-normalizer==3.4.3 +llvmlite==0.44.0 +safetensors==0.6.2 +fqdn==1.5.1 +peft==0.17.0 +fastapi-cli==0.0.8 +torch==2.7.1 +jsonschema==4.25.0 +torchvision==0.22.1 +parso==0.8.4 +pillow==11.3.0 +pycountry==24.6.1 +scipy==1.16.1 +wandb==0.21.1 +pip==25.1 +wcwidth==0.2.13 +mistral_common==1.8.3 +nvidia-cudnn-cu12==9.5.1.17 +asttokens==3.0.0 +numpy==2.2.6 +uvicorn==0.35.0 +jupyter_client==8.6.3 +debugpy==1.8.16 +fastapi-cloud-cli==0.1.5 +compressed-tensors==0.10.2 +GitPython==3.1.45 +nvidia-ml-py==13.580.65 +tiktoken==0.11.0 +partial-json-parser==0.2.1.1.post6 +prompt_toolkit==3.0.51 +uri-template==1.3.0 +pyarrow==21.0.0 +nvidia-cuda-nvrtc-cu12==12.6.77 +propcache==0.3.2 +matplotlib-inline==0.1.7 +pure_eval==0.2.3 +nest-asyncio==1.6.0 +deepspeed==0.17.4 +einops==0.8.1 +triton==3.3.1 +trl==0.21.0 +pyzmq==27.0.1 +httptools==0.6.4 +ipython==9.4.0 +packaging==25.0 +hjson==3.1.0 +soxr==0.5.0.post1 +webcolors==24.11.1 +PyYAML==6.0.2 +ipywidgets==8.1.7 +lark==1.2.2 +python-multipart==0.0.20 +tinycss2==1.4.0 +mpmath==1.3.0 +psutil==7.0.0 +nbformat==5.10.4 +qwen-vl-utils==0.0.11 +notebook==7.4.5 +jupyterlab==4.4.5 +transformers==4.55.0 +urllib3==2.5.0 +terminado==0.18.1 +xgrammar==0.1.21 +xxhash==3.5.0 +tzdata==2025.2 +jupyterlab_widgets==3.0.15 +aiosignal==1.4.0 +annotated-types==0.7.0 +nvidia-nvjitlink-cu12==12.6.85 +httpcore==1.0.9 +pexpect==4.9.0 +tornado==6.5.2 +regex==2025.7.33 +filelock==3.18.0 +nvidia-cuda-runtime-cu12==12.6.77 +openai==1.90.0 +Pygments==2.19.2 +cffi==1.17.1 +nvidia-cufft-cu12==11.3.0.4 +rpds-py==0.27.0 +rich-toolkit==0.15.0 +bleach==6.2.0 +sympy==1.14.0 +python-dateutil==2.9.0.post0 +watchfiles==1.1.0 +nbclient==0.10.2 +hf-xet==1.1.7 +Send2Trash==1.8.3 +smmap==5.0.2 +tokenizers==0.21.4 +yarl==1.20.1 +MarkupSafe==3.0.2 +prometheus-fastapi-instrumentator==7.1.0 +rfc3986-validator==0.1.1 +bitsandbytes==0.46.1 +nvidia-cufile-cu12==1.11.1.6 +fsspec==2025.3.0 +executing==2.2.0 +pydantic==2.11.7 +jupyter_core==5.8.1 +msgspec==0.19.0 +nvidia-cusparse-cu12==12.5.4.2 +opencv-python-headless==4.12.0.88 +webencodings==0.5.1 +ptyprocess==0.7.0 +pytorch-lightning==2.5.2 +Jinja2==3.1.6 +websocket-client==1.8.0 +jsonpointer==3.0.0 +traitlets==5.14.3 +numba==0.61.2 +soupsieve==2.7 +cbor2==5.6.5 +multiprocess==0.70.16 +email_validator==2.2.0 +distro==1.9.0 +accelerate==1.10.0 +torchaudio==2.7.1 +jiter==0.10.0 +rfc3339-validator==0.1.4 +jupyter==1.1.1 +typer==0.16.0 +gitdb==4.0.12 +types-python-dateutil==2.9.0.20250809 +soundfile==0.13.1 +astor==0.8.1 +shellingham==1.5.4 +jupyterlab_server==2.27.3 +nvidia-curand-cu12==10.3.7.77 +jupyter-console==6.6.3 +msgpack==1.1.1 +six==1.17.0 +pycparser==2.22 +arrow==1.3.0 +tqdm==4.67.1 +lightning-utilities==0.15.2 +interegular==0.3.3 +rfc3987-syntax==1.1.0 +ipython_pygments_lexers==1.1.1 +datasets==4.0.0 +jupyter_server_terminals==0.5.3 +platformdirs==4.3.8 +pydantic-extra-types==2.10.5 +decorator==5.2.1 +mdurl==0.1.2 +nvidia-cublas-cu12==12.6.4.1 +jsonschema-specifications==2025.4.1 +setuptools==78.1.1 +fastrlock==0.8.3 +argon2-cffi-bindings==25.1.0 +nbconvert==7.16.6 diff --git a/BioReason_new/wandb/run-20250813_125236-re7fb96i/files/wandb-metadata.json b/BioReason_new/wandb/run-20250813_125236-re7fb96i/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..9c7ca7746ccf900312c8a358d6671e03c8a0d793 --- /dev/null +++ b/BioReason_new/wandb/run-20250813_125236-re7fb96i/files/wandb-metadata.json @@ -0,0 +1,116 @@ +{ + "os": "Linux-5.10.134-008.18.kangaroo.al8.x86_64-x86_64-with-glibc2.35", + "python": "CPython 3.11.0", + "startedAt": "2025-08-13T04:52:36.303807Z", + "args": [ + "--text_model_name", + "/oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged", + "--protein_model_name", + "/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m", + "--qformer_model_name", + "/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft", + "--num_query_tokens", + "8", + "--train_dataset", + "/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl", + "--valid_dataset", + "/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl", + "--output_dir", + "./contrastive_outputs", + "--num_epochs", + "10", + "--batch_size", + "8", + "--learning_rate", + "1e-4", + "--temperature", + "0.07", + "--freeze_protein_model", + "--freeze_text_model", + "--enable_ptm", + "--max_length_protein", + "1024", + "--max_length_text", + "512", + "--num_workers", + "8", + "--eval_dataset", + "--use_wandb", + "--wandb_project", + "protein-llm-contrastive", + "--logging_steps", + "100", + "--eval_steps", + "500", + "--save_steps", + "1000" + ], + "program": "/nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py", + "codePath": "wangyujia/BioReason_new/train_contrastive.py", + "codePathLocal": "train_contrastive.py", + "git": { + "remote": "https://github.com/PorUna-byte/PAR.git", + "commit": "b8caf406aa1699c788f0ca6e44a1769452c317db" + }, + "email": "gia0603yucca@gmail.com", + "root": "/nas/shared/kilab/wangyujia/BioReason_new", + "host": "dsw-265304-585cc9d768-ckfd7", + "executable": "/root/miniconda3/envs/bioreason/bin/python", + "cpu_count": 64, + "cpu_count_logical": 64, + "gpu": "NVIDIA A800-SXM4-80GB", + "gpu_count": 8, + "disk": { + "/": { + "total": "1623302262784", + "used": "1015009280" + } + }, + "memory": { + "total": "549755813888" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-62fcc532-880f-7324-eb13-3b739a38c29b" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-af1197aa-3e7d-a53e-185d-04b1d64164da" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-3e6e551f-2d7d-a27b-654a-da56933d2283" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-a947b64a-c545-ee5e-30e6-b4b0e2dd8fdc" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-ebd04f3f-c437-1b8b-26f2-4b1ebfee93c4" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-53994891-72a6-25be-a4c8-e32c695155b0" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-a927d00a-15c5-31f5-9ae1-9d2e773821a1" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-93d555c4-29a0-cd78-abac-d98cd160d4e1" + } + ], + "cudaVersion": "12.1", + "writerId": "gef9d42v3ap5mwuxow19xbx613zovmhi" +} \ No newline at end of file diff --git a/BioReason_new/wandb/run-20250813_125236-re7fb96i/files/wandb-summary.json b/BioReason_new/wandb/run-20250813_125236-re7fb96i/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..b83728ff622654da73d628f469e3fbb77e031a30 --- /dev/null +++ b/BioReason_new/wandb/run-20250813_125236-re7fb96i/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":14},"_runtime":14} \ No newline at end of file diff --git a/BioReason_new/wandb/run-20250813_125236-re7fb96i/logs/debug-internal.log b/BioReason_new/wandb/run-20250813_125236-re7fb96i/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..41ee7001c19933cd3547e31c42af72271e15bac9 --- /dev/null +++ b/BioReason_new/wandb/run-20250813_125236-re7fb96i/logs/debug-internal.log @@ -0,0 +1,18 @@ +{"time":"2025-08-13T12:52:36.589616991+08:00","level":"INFO","msg":"stream: starting","core version":"0.21.1"} +{"time":"2025-08-13T12:52:38.561487318+08:00","level":"INFO","msg":"stream: created new stream","id":"re7fb96i"} +{"time":"2025-08-13T12:52:38.561535116+08:00","level":"INFO","msg":"stream: started","id":"re7fb96i"} +{"time":"2025-08-13T12:52:38.56159066+08:00","level":"INFO","msg":"writer: started","stream_id":"re7fb96i"} +{"time":"2025-08-13T12:52:38.561602542+08:00","level":"INFO","msg":"sender: started","stream_id":"re7fb96i"} +{"time":"2025-08-13T12:52:38.561649125+08:00","level":"INFO","msg":"handler: started","stream_id":"re7fb96i"} +{"time":"2025-08-13T12:52:44.06545967+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/re7fb96i/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T045241Z&X-Goog-Expires=86399&X-Goog-Signature=5ab95aad544e88ddad2af3439c4ef5fd6cad599df4041ec7a3094e1270d666c27d4c5d84bd7bae3293680630f98ef7873f55e59c7cbb9c983f77007539bc165573b1876eafbf04a7703c05db86dbf5d62862b67515b993edec4fb89877428a0f6ff39beefc2c2d965c832b1f9bbd8cd7ee52831df86c571243838706679984faa939e743fa1c888f4a60beccecf15caba84a74ff0dd96023d8f89371f4b0849496de2439804149a39b3f1dbb3b1a7a3702a46d55b34fd4dc64fc181ad0eaa42bf08cc9856827be672f72262b6c85e3d44dc6efd662c5f2f1f67840978533a27c4c6ed0e131632f8aa8306e955bc031ef6e95e3d62b8eea848f0e2d36ceccfa19&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:53438->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/re7fb96i/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T045241Z&X-Goog-Expires=86399&X-Goog-Signature=5ab95aad544e88ddad2af3439c4ef5fd6cad599df4041ec7a3094e1270d666c27d4c5d84bd7bae3293680630f98ef7873f55e59c7cbb9c983f77007539bc165573b1876eafbf04a7703c05db86dbf5d62862b67515b993edec4fb89877428a0f6ff39beefc2c2d965c832b1f9bbd8cd7ee52831df86c571243838706679984faa939e743fa1c888f4a60beccecf15caba84a74ff0dd96023d8f89371f4b0849496de2439804149a39b3f1dbb3b1a7a3702a46d55b34fd4dc64fc181ad0eaa42bf08cc9856827be672f72262b6c85e3d44dc6efd662c5f2f1f67840978533a27c4c6ed0e131632f8aa8306e955bc031ef6e95e3d62b8eea848f0e2d36ceccfa19&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T12:52:46.994692315+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/re7fb96i/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T045241Z&X-Goog-Expires=86399&X-Goog-Signature=5ab95aad544e88ddad2af3439c4ef5fd6cad599df4041ec7a3094e1270d666c27d4c5d84bd7bae3293680630f98ef7873f55e59c7cbb9c983f77007539bc165573b1876eafbf04a7703c05db86dbf5d62862b67515b993edec4fb89877428a0f6ff39beefc2c2d965c832b1f9bbd8cd7ee52831df86c571243838706679984faa939e743fa1c888f4a60beccecf15caba84a74ff0dd96023d8f89371f4b0849496de2439804149a39b3f1dbb3b1a7a3702a46d55b34fd4dc64fc181ad0eaa42bf08cc9856827be672f72262b6c85e3d44dc6efd662c5f2f1f67840978533a27c4c6ed0e131632f8aa8306e955bc031ef6e95e3d62b8eea848f0e2d36ceccfa19&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:46682->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/re7fb96i/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T045241Z&X-Goog-Expires=86399&X-Goog-Signature=5ab95aad544e88ddad2af3439c4ef5fd6cad599df4041ec7a3094e1270d666c27d4c5d84bd7bae3293680630f98ef7873f55e59c7cbb9c983f77007539bc165573b1876eafbf04a7703c05db86dbf5d62862b67515b993edec4fb89877428a0f6ff39beefc2c2d965c832b1f9bbd8cd7ee52831df86c571243838706679984faa939e743fa1c888f4a60beccecf15caba84a74ff0dd96023d8f89371f4b0849496de2439804149a39b3f1dbb3b1a7a3702a46d55b34fd4dc64fc181ad0eaa42bf08cc9856827be672f72262b6c85e3d44dc6efd662c5f2f1f67840978533a27c4c6ed0e131632f8aa8306e955bc031ef6e95e3d62b8eea848f0e2d36ceccfa19&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T12:52:51.612089813+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/re7fb96i/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T045241Z&X-Goog-Expires=86399&X-Goog-Signature=5ab95aad544e88ddad2af3439c4ef5fd6cad599df4041ec7a3094e1270d666c27d4c5d84bd7bae3293680630f98ef7873f55e59c7cbb9c983f77007539bc165573b1876eafbf04a7703c05db86dbf5d62862b67515b993edec4fb89877428a0f6ff39beefc2c2d965c832b1f9bbd8cd7ee52831df86c571243838706679984faa939e743fa1c888f4a60beccecf15caba84a74ff0dd96023d8f89371f4b0849496de2439804149a39b3f1dbb3b1a7a3702a46d55b34fd4dc64fc181ad0eaa42bf08cc9856827be672f72262b6c85e3d44dc6efd662c5f2f1f67840978533a27c4c6ed0e131632f8aa8306e955bc031ef6e95e3d62b8eea848f0e2d36ceccfa19&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:57324->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/re7fb96i/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T045241Z&X-Goog-Expires=86399&X-Goog-Signature=5ab95aad544e88ddad2af3439c4ef5fd6cad599df4041ec7a3094e1270d666c27d4c5d84bd7bae3293680630f98ef7873f55e59c7cbb9c983f77007539bc165573b1876eafbf04a7703c05db86dbf5d62862b67515b993edec4fb89877428a0f6ff39beefc2c2d965c832b1f9bbd8cd7ee52831df86c571243838706679984faa939e743fa1c888f4a60beccecf15caba84a74ff0dd96023d8f89371f4b0849496de2439804149a39b3f1dbb3b1a7a3702a46d55b34fd4dc64fc181ad0eaa42bf08cc9856827be672f72262b6c85e3d44dc6efd662c5f2f1f67840978533a27c4c6ed0e131632f8aa8306e955bc031ef6e95e3d62b8eea848f0e2d36ceccfa19&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T12:52:55.303102498+08:00","level":"INFO","msg":"stream: closing","id":"re7fb96i"} +{"time":"2025-08-13T12:52:57.319008656+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/re7fb96i/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T045256Z&X-Goog-Expires=86399&X-Goog-Signature=6cd70ba74845b1ae0f9a8f83f1a4441043fa503279d42e4f7e55f638f643835d7270b1d0c42f0cb81df5618f3250c30a10fd02484a7951eeaac60df9b1912a487c976b34dc34f276fd38be0bd372a32e1951ea232361be43a8f78feb4c52042f05a5e7fe515d2e96d6e6187378eee4ff85ed4df1639c554abddd23693fc3c951722445f98a3d6e62052d0c35daa557227cd76aa9efbebe6191e1ae0d7b4ed7241abeab321d09c60a591aeed73d864d6b86496848ab61b6517e66eac19ec2ceaebdb5ae43c1010b820eba8b9e2694683b136e639c2186f348671a910a2f1fa0121439bafd138ff38092414668a7fb59e535b1808cefa7fc42747b4b7b3ba26bc5&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:41368->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/re7fb96i/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T045256Z&X-Goog-Expires=86399&X-Goog-Signature=6cd70ba74845b1ae0f9a8f83f1a4441043fa503279d42e4f7e55f638f643835d7270b1d0c42f0cb81df5618f3250c30a10fd02484a7951eeaac60df9b1912a487c976b34dc34f276fd38be0bd372a32e1951ea232361be43a8f78feb4c52042f05a5e7fe515d2e96d6e6187378eee4ff85ed4df1639c554abddd23693fc3c951722445f98a3d6e62052d0c35daa557227cd76aa9efbebe6191e1ae0d7b4ed7241abeab321d09c60a591aeed73d864d6b86496848ab61b6517e66eac19ec2ceaebdb5ae43c1010b820eba8b9e2694683b136e639c2186f348671a910a2f1fa0121439bafd138ff38092414668a7fb59e535b1808cefa7fc42747b4b7b3ba26bc5&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T12:54:51.198748634+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/re7fb96i/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T045241Z&X-Goog-Expires=86399&X-Goog-Signature=66a468dbcc807521a33850cbde9ed7c890df5238e496201fe7c802709c40b88899c9145fa156b9e5ff9d60d4fce2e40a3550b45198637bc47d62374ab504d81be8116a0d981e038945eb827e976a9e196c72459cd5324c1f95a29ca9f03c9448c37efe4c69f9fa20e463fca0383d500a543ff5a0d68c4b29831d61803b4640eb4ead83681cbfd46b82dbbef572a174fc49c715ff3886d78f6490b7d8162b8479cb62884c5951ff89d46f926ecee4548e03b9460e45e0f3dade083311e0770dcaa8252111666869262f4647000463a4c49491d12f9aa50488cf2d22d1809f5b77a00e9ee23657fcc7420c084261ed55d8370ac97e8f523eb950a7827b25ec4736&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:53304->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/re7fb96i/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T045241Z&X-Goog-Expires=86399&X-Goog-Signature=66a468dbcc807521a33850cbde9ed7c890df5238e496201fe7c802709c40b88899c9145fa156b9e5ff9d60d4fce2e40a3550b45198637bc47d62374ab504d81be8116a0d981e038945eb827e976a9e196c72459cd5324c1f95a29ca9f03c9448c37efe4c69f9fa20e463fca0383d500a543ff5a0d68c4b29831d61803b4640eb4ead83681cbfd46b82dbbef572a174fc49c715ff3886d78f6490b7d8162b8479cb62884c5951ff89d46f926ecee4548e03b9460e45e0f3dade083311e0770dcaa8252111666869262f4647000463a4c49491d12f9aa50488cf2d22d1809f5b77a00e9ee23657fcc7420c084261ed55d8370ac97e8f523eb950a7827b25ec4736&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T12:54:53.616083501+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/re7fb96i/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T045241Z&X-Goog-Expires=86399&X-Goog-Signature=66a468dbcc807521a33850cbde9ed7c890df5238e496201fe7c802709c40b88899c9145fa156b9e5ff9d60d4fce2e40a3550b45198637bc47d62374ab504d81be8116a0d981e038945eb827e976a9e196c72459cd5324c1f95a29ca9f03c9448c37efe4c69f9fa20e463fca0383d500a543ff5a0d68c4b29831d61803b4640eb4ead83681cbfd46b82dbbef572a174fc49c715ff3886d78f6490b7d8162b8479cb62884c5951ff89d46f926ecee4548e03b9460e45e0f3dade083311e0770dcaa8252111666869262f4647000463a4c49491d12f9aa50488cf2d22d1809f5b77a00e9ee23657fcc7420c084261ed55d8370ac97e8f523eb950a7827b25ec4736&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:51230->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/re7fb96i/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T045241Z&X-Goog-Expires=86399&X-Goog-Signature=66a468dbcc807521a33850cbde9ed7c890df5238e496201fe7c802709c40b88899c9145fa156b9e5ff9d60d4fce2e40a3550b45198637bc47d62374ab504d81be8116a0d981e038945eb827e976a9e196c72459cd5324c1f95a29ca9f03c9448c37efe4c69f9fa20e463fca0383d500a543ff5a0d68c4b29831d61803b4640eb4ead83681cbfd46b82dbbef572a174fc49c715ff3886d78f6490b7d8162b8479cb62884c5951ff89d46f926ecee4548e03b9460e45e0f3dade083311e0770dcaa8252111666869262f4647000463a4c49491d12f9aa50488cf2d22d1809f5b77a00e9ee23657fcc7420c084261ed55d8370ac97e8f523eb950a7827b25ec4736&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T12:55:07.705724989+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/re7fb96i/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T045256Z&X-Goog-Expires=86399&X-Goog-Signature=21e8bf224738001dd556d09ec67e19b73e3bbf8672f24a2a3d1f83b54e794b40198452cb9f3b3e9e7726111cc5a89448372dc9a2f3c8414180b3ab89bf6dda7eb7ed58bbe92da4fb684762d530af66a5ef72fd8b746c34892dd054fa7fe727660862a6e5f11ace5ce16d47e03034dc5a8f4bd240f3c58691a1f8b093baef3e4029aa382a17fac6d918e2b26fb8a2b395cc3fb091db2e25ae7d683b992ab3828622593250c82968565c439a973999890b21c772f52c76ce6ec08863a755e829a7531cec69482c34a6956b56e7f83ca69dad5e7e270c21b429968530587522425487f8d0b5de513173609a157f46dac948cafdc901306b271f645af87c0f5c0399&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:34228->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/re7fb96i/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T045256Z&X-Goog-Expires=86399&X-Goog-Signature=21e8bf224738001dd556d09ec67e19b73e3bbf8672f24a2a3d1f83b54e794b40198452cb9f3b3e9e7726111cc5a89448372dc9a2f3c8414180b3ab89bf6dda7eb7ed58bbe92da4fb684762d530af66a5ef72fd8b746c34892dd054fa7fe727660862a6e5f11ace5ce16d47e03034dc5a8f4bd240f3c58691a1f8b093baef3e4029aa382a17fac6d918e2b26fb8a2b395cc3fb091db2e25ae7d683b992ab3828622593250c82968565c439a973999890b21c772f52c76ce6ec08863a755e829a7531cec69482c34a6956b56e7f83ca69dad5e7e270c21b429968530587522425487f8d0b5de513173609a157f46dac948cafdc901306b271f645af87c0f5c0399&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T12:55:10.365569392+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-08-13T12:55:12.589325353+08:00","level":"INFO","msg":"handler: closed","stream_id":"re7fb96i"} +{"time":"2025-08-13T12:55:12.5926135+08:00","level":"INFO","msg":"sender: closed","stream_id":"re7fb96i"} +{"time":"2025-08-13T12:55:12.592621763+08:00","level":"INFO","msg":"stream: closed","id":"re7fb96i"} diff --git a/BioReason_new/wandb/run-20250813_125236-re7fb96i/logs/debug.log b/BioReason_new/wandb/run-20250813_125236-re7fb96i/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e0d161520416679e4e1f591ae30ede9d82e0bac3 --- /dev/null +++ b/BioReason_new/wandb/run-20250813_125236-re7fb96i/logs/debug.log @@ -0,0 +1,23 @@ +2025-08-13 12:52:36,370 INFO MainThread:4711 [wandb_setup.py:_flush():80] Current SDK version is 0.21.1 +2025-08-13 12:52:36,370 INFO MainThread:4711 [wandb_setup.py:_flush():80] Configure stats pid to 4711 +2025-08-13 12:52:36,370 INFO MainThread:4711 [wandb_setup.py:_flush():80] Loading settings from /root/.config/wandb/settings +2025-08-13 12:52:36,370 INFO MainThread:4711 [wandb_setup.py:_flush():80] Loading settings from /nas/shared/kilab/wangyujia/BioReason_new/wandb/settings +2025-08-13 12:52:36,370 INFO MainThread:4711 [wandb_setup.py:_flush():80] Loading settings from environment variables +2025-08-13 12:52:36,370 INFO MainThread:4711 [wandb_init.py:setup_run_log_directory():703] Logging user logs to /nas/shared/kilab/wangyujia/BioReason_new/wandb/run-20250813_125236-re7fb96i/logs/debug.log +2025-08-13 12:52:36,370 INFO MainThread:4711 [wandb_init.py:setup_run_log_directory():704] Logging internal logs to /nas/shared/kilab/wangyujia/BioReason_new/wandb/run-20250813_125236-re7fb96i/logs/debug-internal.log +2025-08-13 12:52:36,370 INFO MainThread:4711 [wandb_init.py:init():830] calling init triggers +2025-08-13 12:52:36,370 INFO MainThread:4711 [wandb_init.py:init():835] wandb.init called with sweep_config: {} +config: {'text_model_name': '/oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged', 'protein_model_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m', 'qformer_model_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft', 'cache_dir': '/model-weights', 'num_query_tokens': 8, 'train_dataset': '/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl', 'valid_dataset': '/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl', 'eval_dataset': True, 'output_dir': './contrastive_outputs', 'num_epochs': 10, 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 0.01, 'warmup_steps': 1000, 'gradient_accumulation_steps': 1, 'temperature': 0.07, 'freeze_protein_model': True, 'freeze_text_model': True, 'protein_weight': 1.0, 'text_weight': 1.0, 'enable_ptm': True, 'ptm_weight': 1.0, 'max_length_protein': 1024, 'max_length_text': 512, 'num_workers': 8, 'logging_steps': 100, 'eval_steps': 500, 'save_steps': 1000, 'save_total_limit': 3, 'fp16': False, 'bf16': False, 'seed': 42, 'use_wandb': True, 'wandb_project': 'protein-llm-contrastive', 'wandb_entity': None, '_wandb': {}} +2025-08-13 12:52:36,370 INFO MainThread:4711 [wandb_init.py:init():871] starting backend +2025-08-13 12:52:36,579 INFO MainThread:4711 [wandb_init.py:init():874] sending inform_init request +2025-08-13 12:52:36,585 INFO MainThread:4711 [wandb_init.py:init():882] backend started and connected +2025-08-13 12:52:36,586 INFO MainThread:4711 [wandb_init.py:init():953] updated telemetry +2025-08-13 12:52:36,614 INFO MainThread:4711 [wandb_init.py:init():977] communicating run to backend with 90.0 second timeout +2025-08-13 12:52:41,160 INFO MainThread:4711 [wandb_init.py:init():1029] starting run threads in backend +2025-08-13 12:52:41,294 INFO MainThread:4711 [wandb_run.py:_console_start():2494] atexit reg +2025-08-13 12:52:41,294 INFO MainThread:4711 [wandb_run.py:_redirect():2342] redirect: wrap_raw +2025-08-13 12:52:41,294 INFO MainThread:4711 [wandb_run.py:_redirect():2411] Wrapping output streams. +2025-08-13 12:52:41,294 INFO MainThread:4711 [wandb_run.py:_redirect():2434] Redirects installed. +2025-08-13 12:52:41,296 INFO MainThread:4711 [wandb_init.py:init():1075] run started, returning control to user process +2025-08-13 12:52:51,565 INFO MainThread:4711 [wandb_run.py:_config_callback():1380] config_cb None None {'output_dir': './contrastive_outputs', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': True, 'do_predict': False, 'eval_strategy': 'steps', 'prediction_loss_only': False, 'per_device_train_batch_size': 8, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'eval_delay': 0, 'torch_empty_cache_steps': None, 'learning_rate': 0.0001, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 10, 'max_steps': -1, 'lr_scheduler_type': 'linear', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.0, 'warmup_steps': 1000, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': './contrastive_outputs/runs/Aug13_12-52-45_dsw-265304-585cc9d768-ckfd7', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 100, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 1000, 'save_total_limit': 3, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'restore_callback_states_from_checkpoint': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 42, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': False, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 500, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': None, 'disable_tqdm': False, 'remove_unused_columns': False, 'label_names': None, 'load_best_model_at_end': True, 'metric_for_best_model': 'eval_avg_recall_at_1', 'greater_is_better': True, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'deepspeed': None, 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['wandb'], 'ddp_find_unused_parameters': False, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '', 'hub_private_repo': None, 'hub_always_push': False, 'hub_revision': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'include_for_metrics': [], 'eval_do_concat_batches': True, 'fp16_backend': 'auto', 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None, 'batch_eval_metrics': False, 'eval_on_start': False, 'use_liger_kernel': False, 'liger_kernel_config': None, 'eval_use_gather_object': False, 'average_tokens_across_devices': False, 'temperature': 0.07, 'freeze_protein_model': True, 'freeze_text_model': True, 'protein_weight': 1.0, 'text_weight': 1.0, 'max_length_protein': 1024, 'max_length_text': 512, 'enable_ptm': True, 'ptm_weight': 1.0} +2025-08-13 12:52:55,302 INFO MsgRouterThr:4711 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles. diff --git a/BioReason_new/wandb/run-20250813_125236-re7fb96i/run-re7fb96i.wandb b/BioReason_new/wandb/run-20250813_125236-re7fb96i/run-re7fb96i.wandb new file mode 100644 index 0000000000000000000000000000000000000000..5a3f787f9dc9b2e082dd72a240c5ce6b3d0552f6 Binary files /dev/null and b/BioReason_new/wandb/run-20250813_125236-re7fb96i/run-re7fb96i.wandb differ diff --git a/BioReason_new/wandb/run-20250813_141819-6d95o7hw/files/config.yaml b/BioReason_new/wandb/run-20250813_141819-6d95o7hw/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6177f954cef630b6753cb972060dbec9527fbb88 --- /dev/null +++ b/BioReason_new/wandb/run-20250813_141819-6d95o7hw/files/config.yaml @@ -0,0 +1,458 @@ +_wandb: + value: + cli_version: 0.21.1 + e: + 783suewtq8mq4vax242rrsp73rsqcggf: + args: + - --text_model_name + - /oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged + - --protein_model_name + - /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m + - --qformer_model_name + - /nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft + - --num_query_tokens + - "8" + - --train_dataset + - /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl + - --valid_dataset + - /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl + - --output_dir + - ./contrastive_outputs + - --num_epochs + - "10" + - --batch_size + - "8" + - --learning_rate + - "1e-4" + - --temperature + - "0.07" + - --freeze_protein_model + - --freeze_text_model + - --enable_ptm + - --max_length_protein + - "1024" + - --max_length_text + - "512" + - --num_workers + - "8" + - --eval_dataset + - --use_wandb + - --wandb_project + - protein-llm-contrastive + - --logging_steps + - "100" + - --eval_steps + - "500" + - --save_steps + - "1000" + codePath: wangyujia/BioReason_new/train_contrastive.py + codePathLocal: train_contrastive.py + cpu_count: 64 + cpu_count_logical: 64 + cudaVersion: "12.1" + disk: + /: + total: "1623302262784" + used: "1015054336" + email: gia0603yucca@gmail.com + executable: /root/miniconda3/envs/bioreason/bin/python + git: + commit: b8caf406aa1699c788f0ca6e44a1769452c317db + remote: https://github.com/PorUna-byte/PAR.git + gpu: NVIDIA A800-SXM4-80GB + gpu_count: 8 + gpu_nvidia: + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-62fcc532-880f-7324-eb13-3b739a38c29b + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-af1197aa-3e7d-a53e-185d-04b1d64164da + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-3e6e551f-2d7d-a27b-654a-da56933d2283 + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-a947b64a-c545-ee5e-30e6-b4b0e2dd8fdc + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-ebd04f3f-c437-1b8b-26f2-4b1ebfee93c4 + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-53994891-72a6-25be-a4c8-e32c695155b0 + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-a927d00a-15c5-31f5-9ae1-9d2e773821a1 + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-93d555c4-29a0-cd78-abac-d98cd160d4e1 + host: dsw-265304-585cc9d768-ckfd7 + memory: + total: "549755813888" + os: Linux-5.10.134-008.18.kangaroo.al8.x86_64-x86_64-with-glibc2.35 + program: /nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py + python: CPython 3.11.0 + root: /nas/shared/kilab/wangyujia/BioReason_new + startedAt: "2025-08-13T06:18:19.380426Z" + writerId: 783suewtq8mq4vax242rrsp73rsqcggf + m: + - "1": train/global_step + "6": + - 3 + "7": [] + - "2": '*' + "5": 1 + "6": + - 1 + "7": [] + python_version: 3.11.0 + t: + "1": + - 1 + - 9 + - 11 + - 41 + - 49 + - 51 + - 71 + - 84 + - 98 + - 103 + "2": + - 1 + - 9 + - 11 + - 41 + - 49 + - 51 + - 71 + - 84 + - 98 + - 103 + "3": + - 7 + - 13 + - 16 + - 66 + "4": 3.11.0 + "5": 0.21.1 + "6": 4.55.0 + "9": + "1": transformers_trainer + "12": 0.21.1 + "13": linux-x86_64 +accelerator_config: + value: + dispatch_batches: null + even_batches: true + gradient_accumulation_kwargs: null + non_blocking: false + split_batches: false + use_seedable_sampler: true +adafactor: + value: false +adam_beta1: + value: 0.9 +adam_beta2: + value: 0.999 +adam_epsilon: + value: 1e-08 +auto_find_batch_size: + value: false +average_tokens_across_devices: + value: false +batch_eval_metrics: + value: false +batch_size: + value: 8 +bf16: + value: false +bf16_full_eval: + value: false +cache_dir: + value: /model-weights +data_seed: + value: null +dataloader_drop_last: + value: false +dataloader_num_workers: + value: 8 +dataloader_persistent_workers: + value: false +dataloader_pin_memory: + value: true +dataloader_prefetch_factor: + value: null +ddp_backend: + value: null +ddp_broadcast_buffers: + value: null +ddp_bucket_cap_mb: + value: null +ddp_find_unused_parameters: + value: false +ddp_timeout: + value: 1800 +debug: + value: [] +deepspeed: + value: null +disable_tqdm: + value: false +do_eval: + value: true +do_predict: + value: false +do_train: + value: false +enable_ptm: + value: true +eval_accumulation_steps: + value: null +eval_dataset: + value: true +eval_delay: + value: 0 +eval_do_concat_batches: + value: true +eval_on_start: + value: false +eval_steps: + value: 500 +eval_strategy: + value: steps +eval_use_gather_object: + value: false +fp16: + value: false +fp16_backend: + value: auto +fp16_full_eval: + value: false +fp16_opt_level: + value: O1 +freeze_protein_model: + value: true +freeze_text_model: + value: true +fsdp: + value: [] +fsdp_config: + value: + min_num_params: 0 + xla: false + xla_fsdp_grad_ckpt: false + xla_fsdp_v2: false +fsdp_min_num_params: + value: 0 +fsdp_transformer_layer_cls_to_wrap: + value: null +full_determinism: + value: false +gradient_accumulation_steps: + value: 1 +gradient_checkpointing: + value: false +gradient_checkpointing_kwargs: + value: null +greater_is_better: + value: true +group_by_length: + value: false +half_precision_backend: + value: auto +hub_always_push: + value: false +hub_model_id: + value: null +hub_private_repo: + value: null +hub_revision: + value: null +hub_strategy: + value: every_save +hub_token: + value: +ignore_data_skip: + value: false +include_for_metrics: + value: [] +include_inputs_for_metrics: + value: false +include_num_input_tokens_seen: + value: false +include_tokens_per_second: + value: false +jit_mode_eval: + value: false +label_names: + value: null +label_smoothing_factor: + value: 0 +learning_rate: + value: 0.0001 +length_column_name: + value: length +liger_kernel_config: + value: null +load_best_model_at_end: + value: true +local_rank: + value: 0 +log_level: + value: passive +log_level_replica: + value: warning +log_on_each_node: + value: true +logging_dir: + value: ./contrastive_outputs/runs/Aug13_14-18-29_dsw-265304-585cc9d768-ckfd7 +logging_first_step: + value: false +logging_nan_inf_filter: + value: true +logging_steps: + value: 100 +logging_strategy: + value: steps +lr_scheduler_type: + value: linear +max_grad_norm: + value: 1 +max_length_protein: + value: 1024 +max_length_text: + value: 512 +max_steps: + value: -1 +metric_for_best_model: + value: eval_avg_recall_at_1 +mp_parameters: + value: "" +neftune_noise_alpha: + value: null +no_cuda: + value: false +num_epochs: + value: 10 +num_query_tokens: + value: 8 +num_train_epochs: + value: 10 +num_workers: + value: 8 +optim: + value: adamw_torch +optim_args: + value: null +optim_target_modules: + value: null +output_dir: + value: ./contrastive_outputs +overwrite_output_dir: + value: false +past_index: + value: -1 +per_device_eval_batch_size: + value: 8 +per_device_train_batch_size: + value: 8 +per_gpu_eval_batch_size: + value: null +per_gpu_train_batch_size: + value: null +prediction_loss_only: + value: false +protein_model_name: + value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m +protein_weight: + value: 1 +ptm_weight: + value: 1 +push_to_hub: + value: false +push_to_hub_model_id: + value: null +push_to_hub_organization: + value: null +push_to_hub_token: + value: +qformer_model_name: + value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft +ray_scope: + value: last +remove_unused_columns: + value: false +report_to: + value: + - wandb +restore_callback_states_from_checkpoint: + value: false +resume_from_checkpoint: + value: null +run_name: + value: null +save_on_each_node: + value: false +save_only_model: + value: false +save_safetensors: + value: true +save_steps: + value: 1000 +save_strategy: + value: steps +save_total_limit: + value: 3 +seed: + value: 42 +skip_memory_metrics: + value: true +temperature: + value: 0.07 +text_model_name: + value: /oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged +text_weight: + value: 1 +tf32: + value: null +torch_compile: + value: false +torch_compile_backend: + value: null +torch_compile_mode: + value: null +torch_empty_cache_steps: + value: null +torchdynamo: + value: null +tpu_metrics_debug: + value: false +tpu_num_cores: + value: null +train_dataset: + value: /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl +use_cpu: + value: false +use_ipex: + value: false +use_legacy_prediction_loop: + value: false +use_liger_kernel: + value: false +use_mps_device: + value: false +use_wandb: + value: true +valid_dataset: + value: /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl +wandb_entity: + value: null +wandb_project: + value: protein-llm-contrastive +warmup_ratio: + value: 0 +warmup_steps: + value: 1000 +weight_decay: + value: 0.01 diff --git a/BioReason_new/wandb/run-20250813_141819-6d95o7hw/files/output.log b/BioReason_new/wandb/run-20250813_141819-6d95o7hw/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..a3175ffc963499fa5597f9e136ae471db34f90d0 --- /dev/null +++ b/BioReason_new/wandb/run-20250813_141819-6d95o7hw/files/output.log @@ -0,0 +1,45 @@ +Loading model... +Loading checkpoint shards: 100%|██████████████████████████████████████████████████████| 4/4 [00:01<00:00, 2.48it/s] +Some weights of EsmModel were not initialized from the model checkpoint at /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m and are newly initialized: ['pooler.dense.bias', 'pooler.dense.weight'] +You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference. +Loading datasets... +Training dataset size: 430595 +Eval dataset size: 10000 +Initializing enhanced trainer... +[2025-08-13 14:18:33,864] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) +/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/_distutils_hack/__init__.py:53: UserWarning: Reliance on distutils from stdlib is deprecated. Users must rely on setuptools to provide the distutils module. Avoid importing distutils or import setuptools first, and avoid setting SETUPTOOLS_USE_DISTUTILS=stdlib. Register concerns at https://github.com/pypa/setuptools/issues/new?template=distutils-deprecation.yml + warnings.warn( +[2025-08-13 14:18:34,947] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False +Starting enhanced contrastive training... +- Contrastive learning enabled +- Protein-text matching: enabled +- Temperature: 0.07 +- PTM weight: 1.0 + 0%| | 0/67290 [00:00 + trainer = main(args) + ^^^^^^^^^^ + File "/nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py", line 399, in main + trainer.train() + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/trainer.py", line 2238, in train + return inner_training_loop( + ^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/trainer.py", line 2582, in _inner_training_loop + tr_loss_step = self.training_step(model, inputs, num_items_in_batch) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/trainer.py", line 3796, in training_step + loss = self.compute_loss(model, inputs, num_items_in_batch=num_items_in_batch) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/nas/shared/kilab/wangyujia/BioReason_new/bioreason/trainer/contrast_trainer_new.py", line 289, in compute_loss + protein_features = model.get_protein_features(protein_sequences) # [B, num_queries, embed_dim] + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/nas/shared/kilab/wangyujia/BioReason_new/bioreason/models/protein_llm.py", line 429, in get_protein_features + protein_features = self.protein_projection( + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1762, in _call_impl + return forward_call(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +TypeError: QFormerProjector.forward() got an unexpected keyword argument 'token_type_ids' diff --git a/BioReason_new/wandb/run-20250813_141819-6d95o7hw/files/requirements.txt b/BioReason_new/wandb/run-20250813_141819-6d95o7hw/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..743d03fe27e0b7e200935023c07a1d7fa61b99ce --- /dev/null +++ b/BioReason_new/wandb/run-20250813_141819-6d95o7hw/files/requirements.txt @@ -0,0 +1,233 @@ +jupyterlab_pygments==0.3.0 +depyf==0.19.0 +av==15.0.0 +fastapi==0.116.1 +ray==2.48.0 +cachetools==6.1.0 +prometheus_client==0.22.1 +requests==2.32.4 +jupyter-lsp==2.2.6 +jupyter-events==0.12.0 +anyio==4.10.0 +pybase64==1.4.2 +fastjsonschema==2.21.1 +cloudpickle==3.1.1 +sentry-sdk==2.34.1 +torchmetrics==1.8.1 +wheel==0.45.1 +idna==3.10 +defusedxml==0.7.1 +starlette==0.47.2 +python-dotenv==1.1.1 +comm==0.2.3 +multidict==6.6.3 +python-json-logger==3.3.0 +jupyter_server==2.16.0 +nvidia-nvtx-cu12==12.6.77 +websockets==15.0.1 +referencing==0.36.2 +gguf==0.17.1 +lm-format-enforcer==0.10.12 +xformers==0.0.31 +ninja==1.11.1.4 +beautifulsoup4==4.13.4 +dnspython==2.7.0 +certifi==2025.8.3 +markdown-it-py==3.0.0 +stack-data==0.6.3 +nvidia-cusparselt-cu12==0.6.3 +networkx==3.5 +aiohappyeyeballs==2.6.1 +nvidia-cuda-cupti-cu12==12.6.80 +llguidance==0.7.30 +rich==14.1.0 +outlines_core==0.2.10 +nvidia-nccl-cu12==2.26.2 +pandas==2.3.1 +pydantic_core==2.33.2 +argon2-cffi==25.1.0 +babel==2.17.0 +isoduration==20.11.0 +uvloop==0.21.0 +json5==0.12.0 +widgetsnbextension==4.0.14 +aiohttp==3.12.15 +notebook_shim==0.2.4 +pandocfilters==1.5.1 +vllm==0.10.0 +blake3==1.0.5 +diskcache==5.6.3 +h11==0.16.0 +attrs==25.3.0 +nvidia-cusolver-cu12==11.7.1.2 +click==8.2.1 +cupy-cuda12x==13.5.1 +jedi==0.19.2 +py-cpuinfo==9.0.0 +overrides==7.7.0 +pytz==2025.2 +protobuf==6.31.1 +dill==0.3.8 +ipykernel==6.30.1 +frozenlist==1.7.0 +huggingface-hub==0.34.4 +rignore==0.6.4 +mistune==3.1.3 +sniffio==1.3.1 +httpx==0.28.1 +sentencepiece==0.2.0 +typing_extensions==4.14.1 +typing-inspection==0.4.1 +async-lru==2.0.5 +bioreason==0.1.0 +charset-normalizer==3.4.3 +llvmlite==0.44.0 +safetensors==0.6.2 +fqdn==1.5.1 +peft==0.17.0 +fastapi-cli==0.0.8 +torch==2.7.1 +jsonschema==4.25.0 +torchvision==0.22.1 +parso==0.8.4 +pillow==11.3.0 +pycountry==24.6.1 +scipy==1.16.1 +wandb==0.21.1 +pip==25.1 +wcwidth==0.2.13 +mistral_common==1.8.3 +nvidia-cudnn-cu12==9.5.1.17 +asttokens==3.0.0 +numpy==2.2.6 +uvicorn==0.35.0 +jupyter_client==8.6.3 +debugpy==1.8.16 +fastapi-cloud-cli==0.1.5 +compressed-tensors==0.10.2 +GitPython==3.1.45 +nvidia-ml-py==13.580.65 +tiktoken==0.11.0 +partial-json-parser==0.2.1.1.post6 +prompt_toolkit==3.0.51 +uri-template==1.3.0 +pyarrow==21.0.0 +nvidia-cuda-nvrtc-cu12==12.6.77 +propcache==0.3.2 +matplotlib-inline==0.1.7 +pure_eval==0.2.3 +nest-asyncio==1.6.0 +deepspeed==0.17.4 +einops==0.8.1 +triton==3.3.1 +trl==0.21.0 +pyzmq==27.0.1 +httptools==0.6.4 +ipython==9.4.0 +packaging==25.0 +hjson==3.1.0 +soxr==0.5.0.post1 +webcolors==24.11.1 +PyYAML==6.0.2 +ipywidgets==8.1.7 +lark==1.2.2 +python-multipart==0.0.20 +tinycss2==1.4.0 +mpmath==1.3.0 +psutil==7.0.0 +nbformat==5.10.4 +qwen-vl-utils==0.0.11 +notebook==7.4.5 +jupyterlab==4.4.5 +transformers==4.55.0 +urllib3==2.5.0 +terminado==0.18.1 +xgrammar==0.1.21 +xxhash==3.5.0 +tzdata==2025.2 +jupyterlab_widgets==3.0.15 +aiosignal==1.4.0 +annotated-types==0.7.0 +nvidia-nvjitlink-cu12==12.6.85 +httpcore==1.0.9 +pexpect==4.9.0 +tornado==6.5.2 +regex==2025.7.33 +filelock==3.18.0 +nvidia-cuda-runtime-cu12==12.6.77 +openai==1.90.0 +Pygments==2.19.2 +cffi==1.17.1 +nvidia-cufft-cu12==11.3.0.4 +rpds-py==0.27.0 +rich-toolkit==0.15.0 +bleach==6.2.0 +sympy==1.14.0 +python-dateutil==2.9.0.post0 +watchfiles==1.1.0 +nbclient==0.10.2 +hf-xet==1.1.7 +Send2Trash==1.8.3 +smmap==5.0.2 +tokenizers==0.21.4 +yarl==1.20.1 +MarkupSafe==3.0.2 +prometheus-fastapi-instrumentator==7.1.0 +rfc3986-validator==0.1.1 +bitsandbytes==0.46.1 +nvidia-cufile-cu12==1.11.1.6 +fsspec==2025.3.0 +executing==2.2.0 +pydantic==2.11.7 +jupyter_core==5.8.1 +msgspec==0.19.0 +nvidia-cusparse-cu12==12.5.4.2 +opencv-python-headless==4.12.0.88 +webencodings==0.5.1 +ptyprocess==0.7.0 +pytorch-lightning==2.5.2 +Jinja2==3.1.6 +websocket-client==1.8.0 +jsonpointer==3.0.0 +traitlets==5.14.3 +numba==0.61.2 +soupsieve==2.7 +cbor2==5.6.5 +multiprocess==0.70.16 +email_validator==2.2.0 +distro==1.9.0 +accelerate==1.10.0 +torchaudio==2.7.1 +jiter==0.10.0 +rfc3339-validator==0.1.4 +jupyter==1.1.1 +typer==0.16.0 +gitdb==4.0.12 +types-python-dateutil==2.9.0.20250809 +soundfile==0.13.1 +astor==0.8.1 +shellingham==1.5.4 +jupyterlab_server==2.27.3 +nvidia-curand-cu12==10.3.7.77 +jupyter-console==6.6.3 +msgpack==1.1.1 +six==1.17.0 +pycparser==2.22 +arrow==1.3.0 +tqdm==4.67.1 +lightning-utilities==0.15.2 +interegular==0.3.3 +rfc3987-syntax==1.1.0 +ipython_pygments_lexers==1.1.1 +datasets==4.0.0 +jupyter_server_terminals==0.5.3 +platformdirs==4.3.8 +pydantic-extra-types==2.10.5 +decorator==5.2.1 +mdurl==0.1.2 +nvidia-cublas-cu12==12.6.4.1 +jsonschema-specifications==2025.4.1 +setuptools==78.1.1 +fastrlock==0.8.3 +argon2-cffi-bindings==25.1.0 +nbconvert==7.16.6 diff --git a/BioReason_new/wandb/run-20250813_141819-6d95o7hw/files/wandb-metadata.json b/BioReason_new/wandb/run-20250813_141819-6d95o7hw/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..84bf0607129d258ef62497d73a43c2adccc920c4 --- /dev/null +++ b/BioReason_new/wandb/run-20250813_141819-6d95o7hw/files/wandb-metadata.json @@ -0,0 +1,116 @@ +{ + "os": "Linux-5.10.134-008.18.kangaroo.al8.x86_64-x86_64-with-glibc2.35", + "python": "CPython 3.11.0", + "startedAt": "2025-08-13T06:18:19.380426Z", + "args": [ + "--text_model_name", + "/oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged", + "--protein_model_name", + "/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m", + "--qformer_model_name", + "/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft", + "--num_query_tokens", + "8", + "--train_dataset", + "/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl", + "--valid_dataset", + "/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl", + "--output_dir", + "./contrastive_outputs", + "--num_epochs", + "10", + "--batch_size", + "8", + "--learning_rate", + "1e-4", + "--temperature", + "0.07", + "--freeze_protein_model", + "--freeze_text_model", + "--enable_ptm", + "--max_length_protein", + "1024", + "--max_length_text", + "512", + "--num_workers", + "8", + "--eval_dataset", + "--use_wandb", + "--wandb_project", + "protein-llm-contrastive", + "--logging_steps", + "100", + "--eval_steps", + "500", + "--save_steps", + "1000" + ], + "program": "/nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py", + "codePath": "wangyujia/BioReason_new/train_contrastive.py", + "codePathLocal": "train_contrastive.py", + "git": { + "remote": "https://github.com/PorUna-byte/PAR.git", + "commit": "b8caf406aa1699c788f0ca6e44a1769452c317db" + }, + "email": "gia0603yucca@gmail.com", + "root": "/nas/shared/kilab/wangyujia/BioReason_new", + "host": "dsw-265304-585cc9d768-ckfd7", + "executable": "/root/miniconda3/envs/bioreason/bin/python", + "cpu_count": 64, + "cpu_count_logical": 64, + "gpu": "NVIDIA A800-SXM4-80GB", + "gpu_count": 8, + "disk": { + "/": { + "total": "1623302262784", + "used": "1015054336" + } + }, + "memory": { + "total": "549755813888" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-62fcc532-880f-7324-eb13-3b739a38c29b" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-af1197aa-3e7d-a53e-185d-04b1d64164da" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-3e6e551f-2d7d-a27b-654a-da56933d2283" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-a947b64a-c545-ee5e-30e6-b4b0e2dd8fdc" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-ebd04f3f-c437-1b8b-26f2-4b1ebfee93c4" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-53994891-72a6-25be-a4c8-e32c695155b0" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-a927d00a-15c5-31f5-9ae1-9d2e773821a1" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-93d555c4-29a0-cd78-abac-d98cd160d4e1" + } + ], + "cudaVersion": "12.1", + "writerId": "783suewtq8mq4vax242rrsp73rsqcggf" +} \ No newline at end of file diff --git a/BioReason_new/wandb/run-20250813_141819-6d95o7hw/files/wandb-summary.json b/BioReason_new/wandb/run-20250813_141819-6d95o7hw/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..527c62044a7acaa37985d3799226336eb762803e --- /dev/null +++ b/BioReason_new/wandb/run-20250813_141819-6d95o7hw/files/wandb-summary.json @@ -0,0 +1 @@ +{"_runtime":14,"_wandb":{"runtime":14}} \ No newline at end of file diff --git a/BioReason_new/wandb/run-20250813_141819-6d95o7hw/logs/debug-internal.log b/BioReason_new/wandb/run-20250813_141819-6d95o7hw/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..c82a485df91ac1eaca7f4c491889bb6c29e53a97 --- /dev/null +++ b/BioReason_new/wandb/run-20250813_141819-6d95o7hw/logs/debug-internal.log @@ -0,0 +1,19 @@ +{"time":"2025-08-13T14:18:19.679275905+08:00","level":"INFO","msg":"stream: starting","core version":"0.21.1"} +{"time":"2025-08-13T14:18:23.516936114+08:00","level":"INFO","msg":"stream: created new stream","id":"6d95o7hw"} +{"time":"2025-08-13T14:18:23.517759676+08:00","level":"INFO","msg":"stream: started","id":"6d95o7hw"} +{"time":"2025-08-13T14:18:23.51779428+08:00","level":"INFO","msg":"sender: started","stream_id":"6d95o7hw"} +{"time":"2025-08-13T14:18:23.517793771+08:00","level":"INFO","msg":"writer: started","stream_id":"6d95o7hw"} +{"time":"2025-08-13T14:18:23.51783624+08:00","level":"INFO","msg":"handler: started","stream_id":"6d95o7hw"} +{"time":"2025-08-13T14:18:26.306463347+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/6d95o7hw/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T061824Z&X-Goog-Expires=86399&X-Goog-Signature=368653d66d2fa0c72af20425484a97b74c80c623cfbcd113e7638d58e672856ed1719e829d5fc48438e50516e1a081ff4ee135505f606a06aadfda935ac087e929f22e6edc07e8cf86a943206508b5d114de7866e3e0821e24b7ee872f36b33aa05d0717c89256355d27798137cf600bb5e7c6aa36fabb3ff1a067817560ee05597e6a1d83e088f9158e3280c52aa2bbc1206d9ac932d20a877c3b213397a2da7705231efec42ce78cfcd3d69537d6da25e05fead16f6edb6da6c14b855e35c462e711eb2b6b0edc20eb6bd5d253e99ae0f83dbbac9fb66398eb9c28cfa64f0a4cda60517b87b93ca858d27ff7945af913737b955d2f49799747cbdb2beedc86&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:36176->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/6d95o7hw/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T061824Z&X-Goog-Expires=86399&X-Goog-Signature=368653d66d2fa0c72af20425484a97b74c80c623cfbcd113e7638d58e672856ed1719e829d5fc48438e50516e1a081ff4ee135505f606a06aadfda935ac087e929f22e6edc07e8cf86a943206508b5d114de7866e3e0821e24b7ee872f36b33aa05d0717c89256355d27798137cf600bb5e7c6aa36fabb3ff1a067817560ee05597e6a1d83e088f9158e3280c52aa2bbc1206d9ac932d20a877c3b213397a2da7705231efec42ce78cfcd3d69537d6da25e05fead16f6edb6da6c14b855e35c462e711eb2b6b0edc20eb6bd5d253e99ae0f83dbbac9fb66398eb9c28cfa64f0a4cda60517b87b93ca858d27ff7945af913737b955d2f49799747cbdb2beedc86&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:18:28.860760307+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/6d95o7hw/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T061824Z&X-Goog-Expires=86399&X-Goog-Signature=368653d66d2fa0c72af20425484a97b74c80c623cfbcd113e7638d58e672856ed1719e829d5fc48438e50516e1a081ff4ee135505f606a06aadfda935ac087e929f22e6edc07e8cf86a943206508b5d114de7866e3e0821e24b7ee872f36b33aa05d0717c89256355d27798137cf600bb5e7c6aa36fabb3ff1a067817560ee05597e6a1d83e088f9158e3280c52aa2bbc1206d9ac932d20a877c3b213397a2da7705231efec42ce78cfcd3d69537d6da25e05fead16f6edb6da6c14b855e35c462e711eb2b6b0edc20eb6bd5d253e99ae0f83dbbac9fb66398eb9c28cfa64f0a4cda60517b87b93ca858d27ff7945af913737b955d2f49799747cbdb2beedc86&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:40306->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/6d95o7hw/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T061824Z&X-Goog-Expires=86399&X-Goog-Signature=368653d66d2fa0c72af20425484a97b74c80c623cfbcd113e7638d58e672856ed1719e829d5fc48438e50516e1a081ff4ee135505f606a06aadfda935ac087e929f22e6edc07e8cf86a943206508b5d114de7866e3e0821e24b7ee872f36b33aa05d0717c89256355d27798137cf600bb5e7c6aa36fabb3ff1a067817560ee05597e6a1d83e088f9158e3280c52aa2bbc1206d9ac932d20a877c3b213397a2da7705231efec42ce78cfcd3d69537d6da25e05fead16f6edb6da6c14b855e35c462e711eb2b6b0edc20eb6bd5d253e99ae0f83dbbac9fb66398eb9c28cfa64f0a4cda60517b87b93ca858d27ff7945af913737b955d2f49799747cbdb2beedc86&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:18:38.597075766+08:00","level":"INFO","msg":"stream: closing","id":"6d95o7hw"} +{"time":"2025-08-13T14:18:41.124384983+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/6d95o7hw/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T061839Z&X-Goog-Expires=86399&X-Goog-Signature=37097f0467c2cb0a73f1abeca04122e4d29400711c0c68032c874c811c7fa32f184c9fd288abcafe5538f6df714fd4cdb57a800bca43ecedea9a4987953524ae63497f2c776218d810da53e4e33d6409c5a8152fab28442bafa237041760663a06eb98db75dba3b2f3040076a3abec723cf6dd7c95596ce0915c97d3798df573dafb78c1be90d571173c09d212fda585a3414e5828a4a5d3ee5716d162d784a3954b4d0aa8651a28fbb9864b92a207f9cd6641a23ac16f9a5d4c7a50c5965b353e3bf0d4b1a86f00c37d1530e2f9be58229477567871758828e816cbaf30d2fa684d5f00818dda963017052f6bf7e03fb68c5b195a3a69731b1d9282dbe7cddc&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:57006->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/6d95o7hw/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T061839Z&X-Goog-Expires=86399&X-Goog-Signature=37097f0467c2cb0a73f1abeca04122e4d29400711c0c68032c874c811c7fa32f184c9fd288abcafe5538f6df714fd4cdb57a800bca43ecedea9a4987953524ae63497f2c776218d810da53e4e33d6409c5a8152fab28442bafa237041760663a06eb98db75dba3b2f3040076a3abec723cf6dd7c95596ce0915c97d3798df573dafb78c1be90d571173c09d212fda585a3414e5828a4a5d3ee5716d162d784a3954b4d0aa8651a28fbb9864b92a207f9cd6641a23ac16f9a5d4c7a50c5965b353e3bf0d4b1a86f00c37d1530e2f9be58229477567871758828e816cbaf30d2fa684d5f00818dda963017052f6bf7e03fb68c5b195a3a69731b1d9282dbe7cddc&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:18:42.124134987+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/6d95o7hw/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T061841Z&X-Goog-Expires=86399&X-Goog-Signature=2815ea37d9b1b097f525654a380c6b87428c3213cde6b7197afb098b7d6cbe2a091a67d76b46a30007b5d4a3ae985d776df9d743d51f8b986f8b6e3d1f8648d6cabf09d467065efcfcbc980f395e13a09f3bb3582c483c46b7507551843641ab9a4b695f6fa50b444da9a00a98f6cedc243737a78dc259f25954f50cb54281f837ba898d97069de2bf41f83108e40c4c52980da614fa20f8799fd6474a5445edc9c2b8b593316d25e2ed2ae631b0c39a00f7a5ec0013775ff7e2a32466fa3f935260dc683d0b0acecf7a2250b3aaf031a15c91b9eba92a548979f2e3857c4e1d5f2b2607bb0d909ca9b1d5f759465cc16cd507c7a828a86e771de94e681bd8c4&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:54320->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/6d95o7hw/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T061841Z&X-Goog-Expires=86399&X-Goog-Signature=2815ea37d9b1b097f525654a380c6b87428c3213cde6b7197afb098b7d6cbe2a091a67d76b46a30007b5d4a3ae985d776df9d743d51f8b986f8b6e3d1f8648d6cabf09d467065efcfcbc980f395e13a09f3bb3582c483c46b7507551843641ab9a4b695f6fa50b444da9a00a98f6cedc243737a78dc259f25954f50cb54281f837ba898d97069de2bf41f83108e40c4c52980da614fa20f8799fd6474a5445edc9c2b8b593316d25e2ed2ae631b0c39a00f7a5ec0013775ff7e2a32466fa3f935260dc683d0b0acecf7a2250b3aaf031a15c91b9eba92a548979f2e3857c4e1d5f2b2607bb0d909ca9b1d5f759465cc16cd507c7a828a86e771de94e681bd8c4&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:20:35.777590448+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/6d95o7hw/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T061824Z&X-Goog-Expires=86399&X-Goog-Signature=a4e383bc3740704387f69b5cfd695ead4a69bfe155cc9a50a50eb3ca73735af4d51452984ffe7eb753b3c1c6aa27268975caeb326500d7a5227fe2dbcb9b77f36af4c8a84e285d9679374a7ac036b8565fdac7f706da3cc647b019a8838ae52d1bfb7ecc691c2c3aebd5c3df1d894a6f2a299de849704ad52c1f89f1d7a0a607b7a800e6caac3e331eaa671b38cd87a45c18987fdfce0f65642b8c94fc29fa3222c28e5faf610d144cfad1931f65d3a139551332bcf950b5c7e6dd5a4c29c3fa8469b7f532aa53837a1bc8764b48c318b3af9606fcf4de722e081469dfe99c34c35dd9e563637aa31ca3923ad22535e27c24d6ab52bcb8451026bc37e4bcbe23&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:58284->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/6d95o7hw/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T061824Z&X-Goog-Expires=86399&X-Goog-Signature=a4e383bc3740704387f69b5cfd695ead4a69bfe155cc9a50a50eb3ca73735af4d51452984ffe7eb753b3c1c6aa27268975caeb326500d7a5227fe2dbcb9b77f36af4c8a84e285d9679374a7ac036b8565fdac7f706da3cc647b019a8838ae52d1bfb7ecc691c2c3aebd5c3df1d894a6f2a299de849704ad52c1f89f1d7a0a607b7a800e6caac3e331eaa671b38cd87a45c18987fdfce0f65642b8c94fc29fa3222c28e5faf610d144cfad1931f65d3a139551332bcf950b5c7e6dd5a4c29c3fa8469b7f532aa53837a1bc8764b48c318b3af9606fcf4de722e081469dfe99c34c35dd9e563637aa31ca3923ad22535e27c24d6ab52bcb8451026bc37e4bcbe23&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:20:56.270453021+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/6d95o7hw/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T061841Z&X-Goog-Expires=86399&X-Goog-Signature=2815ea37d9b1b097f525654a380c6b87428c3213cde6b7197afb098b7d6cbe2a091a67d76b46a30007b5d4a3ae985d776df9d743d51f8b986f8b6e3d1f8648d6cabf09d467065efcfcbc980f395e13a09f3bb3582c483c46b7507551843641ab9a4b695f6fa50b444da9a00a98f6cedc243737a78dc259f25954f50cb54281f837ba898d97069de2bf41f83108e40c4c52980da614fa20f8799fd6474a5445edc9c2b8b593316d25e2ed2ae631b0c39a00f7a5ec0013775ff7e2a32466fa3f935260dc683d0b0acecf7a2250b3aaf031a15c91b9eba92a548979f2e3857c4e1d5f2b2607bb0d909ca9b1d5f759465cc16cd507c7a828a86e771de94e681bd8c4&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:37190->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/6d95o7hw/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T061841Z&X-Goog-Expires=86399&X-Goog-Signature=2815ea37d9b1b097f525654a380c6b87428c3213cde6b7197afb098b7d6cbe2a091a67d76b46a30007b5d4a3ae985d776df9d743d51f8b986f8b6e3d1f8648d6cabf09d467065efcfcbc980f395e13a09f3bb3582c483c46b7507551843641ab9a4b695f6fa50b444da9a00a98f6cedc243737a78dc259f25954f50cb54281f837ba898d97069de2bf41f83108e40c4c52980da614fa20f8799fd6474a5445edc9c2b8b593316d25e2ed2ae631b0c39a00f7a5ec0013775ff7e2a32466fa3f935260dc683d0b0acecf7a2250b3aaf031a15c91b9eba92a548979f2e3857c4e1d5f2b2607bb0d909ca9b1d5f759465cc16cd507c7a828a86e771de94e681bd8c4&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:21:00.865375075+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/6d95o7hw/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T061841Z&X-Goog-Expires=86399&X-Goog-Signature=2815ea37d9b1b097f525654a380c6b87428c3213cde6b7197afb098b7d6cbe2a091a67d76b46a30007b5d4a3ae985d776df9d743d51f8b986f8b6e3d1f8648d6cabf09d467065efcfcbc980f395e13a09f3bb3582c483c46b7507551843641ab9a4b695f6fa50b444da9a00a98f6cedc243737a78dc259f25954f50cb54281f837ba898d97069de2bf41f83108e40c4c52980da614fa20f8799fd6474a5445edc9c2b8b593316d25e2ed2ae631b0c39a00f7a5ec0013775ff7e2a32466fa3f935260dc683d0b0acecf7a2250b3aaf031a15c91b9eba92a548979f2e3857c4e1d5f2b2607bb0d909ca9b1d5f759465cc16cd507c7a828a86e771de94e681bd8c4&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:34918->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/6d95o7hw/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T061841Z&X-Goog-Expires=86399&X-Goog-Signature=2815ea37d9b1b097f525654a380c6b87428c3213cde6b7197afb098b7d6cbe2a091a67d76b46a30007b5d4a3ae985d776df9d743d51f8b986f8b6e3d1f8648d6cabf09d467065efcfcbc980f395e13a09f3bb3582c483c46b7507551843641ab9a4b695f6fa50b444da9a00a98f6cedc243737a78dc259f25954f50cb54281f837ba898d97069de2bf41f83108e40c4c52980da614fa20f8799fd6474a5445edc9c2b8b593316d25e2ed2ae631b0c39a00f7a5ec0013775ff7e2a32466fa3f935260dc683d0b0acecf7a2250b3aaf031a15c91b9eba92a548979f2e3857c4e1d5f2b2607bb0d909ca9b1d5f759465cc16cd507c7a828a86e771de94e681bd8c4&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:22:46.942241349+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/6d95o7hw/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T061824Z&X-Goog-Expires=86399&X-Goog-Signature=a4e383bc3740704387f69b5cfd695ead4a69bfe155cc9a50a50eb3ca73735af4d51452984ffe7eb753b3c1c6aa27268975caeb326500d7a5227fe2dbcb9b77f36af4c8a84e285d9679374a7ac036b8565fdac7f706da3cc647b019a8838ae52d1bfb7ecc691c2c3aebd5c3df1d894a6f2a299de849704ad52c1f89f1d7a0a607b7a800e6caac3e331eaa671b38cd87a45c18987fdfce0f65642b8c94fc29fa3222c28e5faf610d144cfad1931f65d3a139551332bcf950b5c7e6dd5a4c29c3fa8469b7f532aa53837a1bc8764b48c318b3af9606fcf4de722e081469dfe99c34c35dd9e563637aa31ca3923ad22535e27c24d6ab52bcb8451026bc37e4bcbe23&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:36662->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/6d95o7hw/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T061824Z&X-Goog-Expires=86399&X-Goog-Signature=a4e383bc3740704387f69b5cfd695ead4a69bfe155cc9a50a50eb3ca73735af4d51452984ffe7eb753b3c1c6aa27268975caeb326500d7a5227fe2dbcb9b77f36af4c8a84e285d9679374a7ac036b8565fdac7f706da3cc647b019a8838ae52d1bfb7ecc691c2c3aebd5c3df1d894a6f2a299de849704ad52c1f89f1d7a0a607b7a800e6caac3e331eaa671b38cd87a45c18987fdfce0f65642b8c94fc29fa3222c28e5faf610d144cfad1931f65d3a139551332bcf950b5c7e6dd5a4c29c3fa8469b7f532aa53837a1bc8764b48c318b3af9606fcf4de722e081469dfe99c34c35dd9e563637aa31ca3923ad22535e27c24d6ab52bcb8451026bc37e4bcbe23&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:22:53.1515727+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-08-13T14:23:06.271059888+08:00","level":"INFO","msg":"handler: closed","stream_id":"6d95o7hw"} +{"time":"2025-08-13T14:23:06.276503172+08:00","level":"INFO","msg":"sender: closed","stream_id":"6d95o7hw"} +{"time":"2025-08-13T14:23:06.276536784+08:00","level":"INFO","msg":"stream: closed","id":"6d95o7hw"} diff --git a/BioReason_new/wandb/run-20250813_141819-6d95o7hw/logs/debug.log b/BioReason_new/wandb/run-20250813_141819-6d95o7hw/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..bea6c79b1f109fad2b39affb4086efa703053315 --- /dev/null +++ b/BioReason_new/wandb/run-20250813_141819-6d95o7hw/logs/debug.log @@ -0,0 +1,23 @@ +2025-08-13 14:18:19,458 INFO MainThread:7754 [wandb_setup.py:_flush():80] Current SDK version is 0.21.1 +2025-08-13 14:18:19,458 INFO MainThread:7754 [wandb_setup.py:_flush():80] Configure stats pid to 7754 +2025-08-13 14:18:19,458 INFO MainThread:7754 [wandb_setup.py:_flush():80] Loading settings from /root/.config/wandb/settings +2025-08-13 14:18:19,458 INFO MainThread:7754 [wandb_setup.py:_flush():80] Loading settings from /nas/shared/kilab/wangyujia/BioReason_new/wandb/settings +2025-08-13 14:18:19,458 INFO MainThread:7754 [wandb_setup.py:_flush():80] Loading settings from environment variables +2025-08-13 14:18:19,458 INFO MainThread:7754 [wandb_init.py:setup_run_log_directory():703] Logging user logs to /nas/shared/kilab/wangyujia/BioReason_new/wandb/run-20250813_141819-6d95o7hw/logs/debug.log +2025-08-13 14:18:19,458 INFO MainThread:7754 [wandb_init.py:setup_run_log_directory():704] Logging internal logs to /nas/shared/kilab/wangyujia/BioReason_new/wandb/run-20250813_141819-6d95o7hw/logs/debug-internal.log +2025-08-13 14:18:19,458 INFO MainThread:7754 [wandb_init.py:init():830] calling init triggers +2025-08-13 14:18:19,458 INFO MainThread:7754 [wandb_init.py:init():835] wandb.init called with sweep_config: {} +config: {'text_model_name': '/oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged', 'protein_model_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m', 'qformer_model_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft', 'cache_dir': '/model-weights', 'num_query_tokens': 8, 'train_dataset': '/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl', 'valid_dataset': '/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl', 'eval_dataset': True, 'output_dir': './contrastive_outputs', 'num_epochs': 10, 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 0.01, 'warmup_steps': 1000, 'gradient_accumulation_steps': 1, 'temperature': 0.07, 'freeze_protein_model': True, 'freeze_text_model': True, 'protein_weight': 1.0, 'text_weight': 1.0, 'enable_ptm': True, 'ptm_weight': 1.0, 'max_length_protein': 1024, 'max_length_text': 512, 'num_workers': 8, 'logging_steps': 100, 'eval_steps': 500, 'save_steps': 1000, 'save_total_limit': 3, 'fp16': False, 'bf16': False, 'seed': 42, 'use_wandb': True, 'wandb_project': 'protein-llm-contrastive', 'wandb_entity': None, '_wandb': {}} +2025-08-13 14:18:19,458 INFO MainThread:7754 [wandb_init.py:init():871] starting backend +2025-08-13 14:18:19,667 INFO MainThread:7754 [wandb_init.py:init():874] sending inform_init request +2025-08-13 14:18:19,673 INFO MainThread:7754 [wandb_init.py:init():882] backend started and connected +2025-08-13 14:18:19,674 INFO MainThread:7754 [wandb_init.py:init():953] updated telemetry +2025-08-13 14:18:19,700 INFO MainThread:7754 [wandb_init.py:init():977] communicating run to backend with 90.0 second timeout +2025-08-13 14:18:24,474 INFO MainThread:7754 [wandb_init.py:init():1029] starting run threads in backend +2025-08-13 14:18:24,583 INFO MainThread:7754 [wandb_run.py:_console_start():2494] atexit reg +2025-08-13 14:18:24,583 INFO MainThread:7754 [wandb_run.py:_redirect():2342] redirect: wrap_raw +2025-08-13 14:18:24,584 INFO MainThread:7754 [wandb_run.py:_redirect():2411] Wrapping output streams. +2025-08-13 14:18:24,584 INFO MainThread:7754 [wandb_run.py:_redirect():2434] Redirects installed. +2025-08-13 14:18:24,586 INFO MainThread:7754 [wandb_init.py:init():1075] run started, returning control to user process +2025-08-13 14:18:35,224 INFO MainThread:7754 [wandb_run.py:_config_callback():1380] config_cb None None {'output_dir': './contrastive_outputs', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': True, 'do_predict': False, 'eval_strategy': 'steps', 'prediction_loss_only': False, 'per_device_train_batch_size': 8, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'eval_delay': 0, 'torch_empty_cache_steps': None, 'learning_rate': 0.0001, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 10, 'max_steps': -1, 'lr_scheduler_type': 'linear', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.0, 'warmup_steps': 1000, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': './contrastive_outputs/runs/Aug13_14-18-29_dsw-265304-585cc9d768-ckfd7', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 100, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 1000, 'save_total_limit': 3, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'restore_callback_states_from_checkpoint': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 42, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': False, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 500, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': None, 'disable_tqdm': False, 'remove_unused_columns': False, 'label_names': None, 'load_best_model_at_end': True, 'metric_for_best_model': 'eval_avg_recall_at_1', 'greater_is_better': True, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'deepspeed': None, 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['wandb'], 'ddp_find_unused_parameters': False, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '', 'hub_private_repo': None, 'hub_always_push': False, 'hub_revision': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'include_for_metrics': [], 'eval_do_concat_batches': True, 'fp16_backend': 'auto', 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None, 'batch_eval_metrics': False, 'eval_on_start': False, 'use_liger_kernel': False, 'liger_kernel_config': None, 'eval_use_gather_object': False, 'average_tokens_across_devices': False, 'temperature': 0.07, 'freeze_protein_model': True, 'freeze_text_model': True, 'protein_weight': 1.0, 'text_weight': 1.0, 'max_length_protein': 1024, 'max_length_text': 512, 'enable_ptm': True, 'ptm_weight': 1.0} +2025-08-13 14:18:38,596 INFO MsgRouterThr:7754 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles. diff --git a/BioReason_new/wandb/run-20250813_141819-6d95o7hw/run-6d95o7hw.wandb b/BioReason_new/wandb/run-20250813_141819-6d95o7hw/run-6d95o7hw.wandb new file mode 100644 index 0000000000000000000000000000000000000000..c3ae28065973c0e2ef3a941b40278dc94f80aec9 Binary files /dev/null and b/BioReason_new/wandb/run-20250813_141819-6d95o7hw/run-6d95o7hw.wandb differ diff --git a/BioReason_new/wandb/run-20250813_142935-iunrz6lf/files/config.yaml b/BioReason_new/wandb/run-20250813_142935-iunrz6lf/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7c793969956930ba5bfe5084c8aae490353d774b --- /dev/null +++ b/BioReason_new/wandb/run-20250813_142935-iunrz6lf/files/config.yaml @@ -0,0 +1,200 @@ +_wandb: + value: + cli_version: 0.21.1 + e: + 4obixt2d5ipbxfjgs2k7zb94y9cqikyp: + args: + - --text_model_name + - /oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged + - --protein_model_name + - /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m + - --qformer_model_name + - /nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft + - --num_query_tokens + - "8" + - --train_dataset + - /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl + - --valid_dataset + - /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl + - --output_dir + - ./contrastive_outputs + - --num_epochs + - "10" + - --batch_size + - "8" + - --learning_rate + - "1e-4" + - --temperature + - "0.07" + - --freeze_protein_model + - --freeze_text_model + - --enable_ptm + - --max_length_protein + - "1024" + - --max_length_text + - "512" + - --num_workers + - "8" + - --eval_dataset + - --use_wandb + - --wandb_project + - protein-llm-contrastive + - --logging_steps + - "100" + - --eval_steps + - "500" + - --save_steps + - "1000" + codePath: wangyujia/BioReason_new/train_contrastive.py + codePathLocal: train_contrastive.py + cpu_count: 64 + cpu_count_logical: 64 + cudaVersion: "12.1" + disk: + /: + total: "1623302262784" + used: "1015083008" + email: gia0603yucca@gmail.com + executable: /root/miniconda3/envs/bioreason/bin/python + git: + commit: b8caf406aa1699c788f0ca6e44a1769452c317db + remote: https://github.com/PorUna-byte/PAR.git + gpu: NVIDIA A800-SXM4-80GB + gpu_count: 8 + gpu_nvidia: + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-62fcc532-880f-7324-eb13-3b739a38c29b + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-af1197aa-3e7d-a53e-185d-04b1d64164da + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-3e6e551f-2d7d-a27b-654a-da56933d2283 + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-a947b64a-c545-ee5e-30e6-b4b0e2dd8fdc + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-ebd04f3f-c437-1b8b-26f2-4b1ebfee93c4 + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-53994891-72a6-25be-a4c8-e32c695155b0 + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-a927d00a-15c5-31f5-9ae1-9d2e773821a1 + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-93d555c4-29a0-cd78-abac-d98cd160d4e1 + host: dsw-265304-585cc9d768-ckfd7 + memory: + total: "549755813888" + os: Linux-5.10.134-008.18.kangaroo.al8.x86_64-x86_64-with-glibc2.35 + program: /nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py + python: CPython 3.11.0 + root: /nas/shared/kilab/wangyujia/BioReason_new + startedAt: "2025-08-13T06:29:35.069039Z" + writerId: 4obixt2d5ipbxfjgs2k7zb94y9cqikyp + m: [] + python_version: 3.11.0 + t: + "1": + - 1 + - 9 + - 11 + - 41 + - 49 + - 51 + - 71 + - 84 + - 98 + - 103 + "2": + - 1 + - 9 + - 11 + - 41 + - 49 + - 51 + - 71 + - 84 + - 98 + - 103 + "3": + - 13 + - 16 + "4": 3.11.0 + "5": 0.21.1 + "6": 4.55.0 + "12": 0.21.1 + "13": linux-x86_64 +batch_size: + value: 8 +bf16: + value: false +cache_dir: + value: /model-weights +enable_ptm: + value: true +eval_dataset: + value: true +eval_steps: + value: 500 +fp16: + value: false +freeze_protein_model: + value: true +freeze_text_model: + value: true +gradient_accumulation_steps: + value: 1 +learning_rate: + value: 0.0001 +logging_steps: + value: 100 +max_length_protein: + value: 1024 +max_length_text: + value: 512 +num_epochs: + value: 10 +num_query_tokens: + value: 8 +num_workers: + value: 8 +output_dir: + value: ./contrastive_outputs +protein_model_name: + value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m +protein_weight: + value: 1 +ptm_weight: + value: 1 +qformer_model_name: + value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft +save_steps: + value: 1000 +save_total_limit: + value: 3 +seed: + value: 42 +temperature: + value: 0.07 +text_model_name: + value: /oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged +text_weight: + value: 1 +train_dataset: + value: /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl +use_wandb: + value: true +valid_dataset: + value: /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl +wandb_entity: + value: null +wandb_project: + value: protein-llm-contrastive +warmup_steps: + value: 1000 +weight_decay: + value: 0.01 diff --git a/BioReason_new/wandb/run-20250813_142935-iunrz6lf/files/output.log b/BioReason_new/wandb/run-20250813_142935-iunrz6lf/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..b873128ee617d0f20ab47fabd8e973e840dcea7a --- /dev/null +++ b/BioReason_new/wandb/run-20250813_142935-iunrz6lf/files/output.log @@ -0,0 +1,40 @@ +Loading model... +Loading checkpoint shards: 100%|██████████████████████████████████████████████████████| 4/4 [00:01<00:00, 2.64it/s] +Some weights of EsmModel were not initialized from the model checkpoint at /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m and are newly initialized: ['pooler.dense.bias', 'pooler.dense.weight'] +You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference. +Traceback (most recent call last): + File "/nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py", line 552, in + trainer = main(args) + ^^^^^^^^^^ + File "/nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py", line 303, in main + model = ProteinLLMModel( + ^^^^^^^^^^^^^^^^ + File "/nas/shared/kilab/wangyujia/BioReason_new/bioreason/models/protein_llm.py", line 290, in __init__ + self.protein_projection = QFormerProjector( + ^^^^^^^^^^^^^^^^^ + File "/nas/shared/kilab/wangyujia/BioReason_new/bioreason/models/protein_llm.py", line 136, in __init__ + self.qformer = BertModel.from_pretrained(qformer_model_name, config=config) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/modeling_utils.py", line 316, in _wrapper + return func(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/modeling_utils.py", line 5061, in from_pretrained + ) = cls._load_pretrained_model( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/modeling_utils.py", line 5524, in _load_pretrained_model + _error_msgs, disk_offload_index, cpu_offload_index = load_shard_file(args) + ^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/modeling_utils.py", line 974, in load_shard_file + disk_offload_index, cpu_offload_index = _load_state_dict_into_meta_model( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context + return func(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/modeling_utils.py", line 879, in _load_state_dict_into_meta_model + _load_parameter_into_model(model, param_name, param.to(param_device)) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/modeling_utils.py", line 739, in _load_parameter_into_model + module.load_state_dict({param_type: tensor}, strict=False, assign=True) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/torch/nn/modules/module.py", line 2593, in load_state_dict + raise RuntimeError( +RuntimeError: Error(s) in loading state_dict for Embedding: + size mismatch for weight: copying a param with shape torch.Size([512, 768]) from checkpoint, the shape in current model is torch.Size([1024, 768]). diff --git a/BioReason_new/wandb/run-20250813_142935-iunrz6lf/files/requirements.txt b/BioReason_new/wandb/run-20250813_142935-iunrz6lf/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..743d03fe27e0b7e200935023c07a1d7fa61b99ce --- /dev/null +++ b/BioReason_new/wandb/run-20250813_142935-iunrz6lf/files/requirements.txt @@ -0,0 +1,233 @@ +jupyterlab_pygments==0.3.0 +depyf==0.19.0 +av==15.0.0 +fastapi==0.116.1 +ray==2.48.0 +cachetools==6.1.0 +prometheus_client==0.22.1 +requests==2.32.4 +jupyter-lsp==2.2.6 +jupyter-events==0.12.0 +anyio==4.10.0 +pybase64==1.4.2 +fastjsonschema==2.21.1 +cloudpickle==3.1.1 +sentry-sdk==2.34.1 +torchmetrics==1.8.1 +wheel==0.45.1 +idna==3.10 +defusedxml==0.7.1 +starlette==0.47.2 +python-dotenv==1.1.1 +comm==0.2.3 +multidict==6.6.3 +python-json-logger==3.3.0 +jupyter_server==2.16.0 +nvidia-nvtx-cu12==12.6.77 +websockets==15.0.1 +referencing==0.36.2 +gguf==0.17.1 +lm-format-enforcer==0.10.12 +xformers==0.0.31 +ninja==1.11.1.4 +beautifulsoup4==4.13.4 +dnspython==2.7.0 +certifi==2025.8.3 +markdown-it-py==3.0.0 +stack-data==0.6.3 +nvidia-cusparselt-cu12==0.6.3 +networkx==3.5 +aiohappyeyeballs==2.6.1 +nvidia-cuda-cupti-cu12==12.6.80 +llguidance==0.7.30 +rich==14.1.0 +outlines_core==0.2.10 +nvidia-nccl-cu12==2.26.2 +pandas==2.3.1 +pydantic_core==2.33.2 +argon2-cffi==25.1.0 +babel==2.17.0 +isoduration==20.11.0 +uvloop==0.21.0 +json5==0.12.0 +widgetsnbextension==4.0.14 +aiohttp==3.12.15 +notebook_shim==0.2.4 +pandocfilters==1.5.1 +vllm==0.10.0 +blake3==1.0.5 +diskcache==5.6.3 +h11==0.16.0 +attrs==25.3.0 +nvidia-cusolver-cu12==11.7.1.2 +click==8.2.1 +cupy-cuda12x==13.5.1 +jedi==0.19.2 +py-cpuinfo==9.0.0 +overrides==7.7.0 +pytz==2025.2 +protobuf==6.31.1 +dill==0.3.8 +ipykernel==6.30.1 +frozenlist==1.7.0 +huggingface-hub==0.34.4 +rignore==0.6.4 +mistune==3.1.3 +sniffio==1.3.1 +httpx==0.28.1 +sentencepiece==0.2.0 +typing_extensions==4.14.1 +typing-inspection==0.4.1 +async-lru==2.0.5 +bioreason==0.1.0 +charset-normalizer==3.4.3 +llvmlite==0.44.0 +safetensors==0.6.2 +fqdn==1.5.1 +peft==0.17.0 +fastapi-cli==0.0.8 +torch==2.7.1 +jsonschema==4.25.0 +torchvision==0.22.1 +parso==0.8.4 +pillow==11.3.0 +pycountry==24.6.1 +scipy==1.16.1 +wandb==0.21.1 +pip==25.1 +wcwidth==0.2.13 +mistral_common==1.8.3 +nvidia-cudnn-cu12==9.5.1.17 +asttokens==3.0.0 +numpy==2.2.6 +uvicorn==0.35.0 +jupyter_client==8.6.3 +debugpy==1.8.16 +fastapi-cloud-cli==0.1.5 +compressed-tensors==0.10.2 +GitPython==3.1.45 +nvidia-ml-py==13.580.65 +tiktoken==0.11.0 +partial-json-parser==0.2.1.1.post6 +prompt_toolkit==3.0.51 +uri-template==1.3.0 +pyarrow==21.0.0 +nvidia-cuda-nvrtc-cu12==12.6.77 +propcache==0.3.2 +matplotlib-inline==0.1.7 +pure_eval==0.2.3 +nest-asyncio==1.6.0 +deepspeed==0.17.4 +einops==0.8.1 +triton==3.3.1 +trl==0.21.0 +pyzmq==27.0.1 +httptools==0.6.4 +ipython==9.4.0 +packaging==25.0 +hjson==3.1.0 +soxr==0.5.0.post1 +webcolors==24.11.1 +PyYAML==6.0.2 +ipywidgets==8.1.7 +lark==1.2.2 +python-multipart==0.0.20 +tinycss2==1.4.0 +mpmath==1.3.0 +psutil==7.0.0 +nbformat==5.10.4 +qwen-vl-utils==0.0.11 +notebook==7.4.5 +jupyterlab==4.4.5 +transformers==4.55.0 +urllib3==2.5.0 +terminado==0.18.1 +xgrammar==0.1.21 +xxhash==3.5.0 +tzdata==2025.2 +jupyterlab_widgets==3.0.15 +aiosignal==1.4.0 +annotated-types==0.7.0 +nvidia-nvjitlink-cu12==12.6.85 +httpcore==1.0.9 +pexpect==4.9.0 +tornado==6.5.2 +regex==2025.7.33 +filelock==3.18.0 +nvidia-cuda-runtime-cu12==12.6.77 +openai==1.90.0 +Pygments==2.19.2 +cffi==1.17.1 +nvidia-cufft-cu12==11.3.0.4 +rpds-py==0.27.0 +rich-toolkit==0.15.0 +bleach==6.2.0 +sympy==1.14.0 +python-dateutil==2.9.0.post0 +watchfiles==1.1.0 +nbclient==0.10.2 +hf-xet==1.1.7 +Send2Trash==1.8.3 +smmap==5.0.2 +tokenizers==0.21.4 +yarl==1.20.1 +MarkupSafe==3.0.2 +prometheus-fastapi-instrumentator==7.1.0 +rfc3986-validator==0.1.1 +bitsandbytes==0.46.1 +nvidia-cufile-cu12==1.11.1.6 +fsspec==2025.3.0 +executing==2.2.0 +pydantic==2.11.7 +jupyter_core==5.8.1 +msgspec==0.19.0 +nvidia-cusparse-cu12==12.5.4.2 +opencv-python-headless==4.12.0.88 +webencodings==0.5.1 +ptyprocess==0.7.0 +pytorch-lightning==2.5.2 +Jinja2==3.1.6 +websocket-client==1.8.0 +jsonpointer==3.0.0 +traitlets==5.14.3 +numba==0.61.2 +soupsieve==2.7 +cbor2==5.6.5 +multiprocess==0.70.16 +email_validator==2.2.0 +distro==1.9.0 +accelerate==1.10.0 +torchaudio==2.7.1 +jiter==0.10.0 +rfc3339-validator==0.1.4 +jupyter==1.1.1 +typer==0.16.0 +gitdb==4.0.12 +types-python-dateutil==2.9.0.20250809 +soundfile==0.13.1 +astor==0.8.1 +shellingham==1.5.4 +jupyterlab_server==2.27.3 +nvidia-curand-cu12==10.3.7.77 +jupyter-console==6.6.3 +msgpack==1.1.1 +six==1.17.0 +pycparser==2.22 +arrow==1.3.0 +tqdm==4.67.1 +lightning-utilities==0.15.2 +interegular==0.3.3 +rfc3987-syntax==1.1.0 +ipython_pygments_lexers==1.1.1 +datasets==4.0.0 +jupyter_server_terminals==0.5.3 +platformdirs==4.3.8 +pydantic-extra-types==2.10.5 +decorator==5.2.1 +mdurl==0.1.2 +nvidia-cublas-cu12==12.6.4.1 +jsonschema-specifications==2025.4.1 +setuptools==78.1.1 +fastrlock==0.8.3 +argon2-cffi-bindings==25.1.0 +nbconvert==7.16.6 diff --git a/BioReason_new/wandb/run-20250813_142935-iunrz6lf/files/wandb-metadata.json b/BioReason_new/wandb/run-20250813_142935-iunrz6lf/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..7bbca5121738d6dd5c3339e911c48ddcc5340523 --- /dev/null +++ b/BioReason_new/wandb/run-20250813_142935-iunrz6lf/files/wandb-metadata.json @@ -0,0 +1,116 @@ +{ + "os": "Linux-5.10.134-008.18.kangaroo.al8.x86_64-x86_64-with-glibc2.35", + "python": "CPython 3.11.0", + "startedAt": "2025-08-13T06:29:35.069039Z", + "args": [ + "--text_model_name", + "/oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged", + "--protein_model_name", + "/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m", + "--qformer_model_name", + "/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft", + "--num_query_tokens", + "8", + "--train_dataset", + "/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl", + "--valid_dataset", + "/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl", + "--output_dir", + "./contrastive_outputs", + "--num_epochs", + "10", + "--batch_size", + "8", + "--learning_rate", + "1e-4", + "--temperature", + "0.07", + "--freeze_protein_model", + "--freeze_text_model", + "--enable_ptm", + "--max_length_protein", + "1024", + "--max_length_text", + "512", + "--num_workers", + "8", + "--eval_dataset", + "--use_wandb", + "--wandb_project", + "protein-llm-contrastive", + "--logging_steps", + "100", + "--eval_steps", + "500", + "--save_steps", + "1000" + ], + "program": "/nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py", + "codePath": "wangyujia/BioReason_new/train_contrastive.py", + "codePathLocal": "train_contrastive.py", + "git": { + "remote": "https://github.com/PorUna-byte/PAR.git", + "commit": "b8caf406aa1699c788f0ca6e44a1769452c317db" + }, + "email": "gia0603yucca@gmail.com", + "root": "/nas/shared/kilab/wangyujia/BioReason_new", + "host": "dsw-265304-585cc9d768-ckfd7", + "executable": "/root/miniconda3/envs/bioreason/bin/python", + "cpu_count": 64, + "cpu_count_logical": 64, + "gpu": "NVIDIA A800-SXM4-80GB", + "gpu_count": 8, + "disk": { + "/": { + "total": "1623302262784", + "used": "1015083008" + } + }, + "memory": { + "total": "549755813888" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-62fcc532-880f-7324-eb13-3b739a38c29b" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-af1197aa-3e7d-a53e-185d-04b1d64164da" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-3e6e551f-2d7d-a27b-654a-da56933d2283" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-a947b64a-c545-ee5e-30e6-b4b0e2dd8fdc" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-ebd04f3f-c437-1b8b-26f2-4b1ebfee93c4" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-53994891-72a6-25be-a4c8-e32c695155b0" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-a927d00a-15c5-31f5-9ae1-9d2e773821a1" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-93d555c4-29a0-cd78-abac-d98cd160d4e1" + } + ], + "cudaVersion": "12.1", + "writerId": "4obixt2d5ipbxfjgs2k7zb94y9cqikyp" +} \ No newline at end of file diff --git a/BioReason_new/wandb/run-20250813_142935-iunrz6lf/files/wandb-summary.json b/BioReason_new/wandb/run-20250813_142935-iunrz6lf/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..75631e6e43bf8471ee6ca35a1ab1286f569677cb --- /dev/null +++ b/BioReason_new/wandb/run-20250813_142935-iunrz6lf/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":3},"_runtime":3} \ No newline at end of file diff --git a/BioReason_new/wandb/run-20250813_142935-iunrz6lf/logs/debug-internal.log b/BioReason_new/wandb/run-20250813_142935-iunrz6lf/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..36bc385d73add3f32678ee34e04e4920e49727a7 --- /dev/null +++ b/BioReason_new/wandb/run-20250813_142935-iunrz6lf/logs/debug-internal.log @@ -0,0 +1,31 @@ +{"time":"2025-08-13T14:29:35.356008701+08:00","level":"INFO","msg":"stream: starting","core version":"0.21.1"} +{"time":"2025-08-13T14:29:38.198330178+08:00","level":"INFO","msg":"stream: created new stream","id":"iunrz6lf"} +{"time":"2025-08-13T14:29:38.198386468+08:00","level":"INFO","msg":"stream: started","id":"iunrz6lf"} +{"time":"2025-08-13T14:29:38.198435503+08:00","level":"INFO","msg":"handler: started","stream_id":"iunrz6lf"} +{"time":"2025-08-13T14:29:38.198417623+08:00","level":"INFO","msg":"writer: started","stream_id":"iunrz6lf"} +{"time":"2025-08-13T14:29:38.19846714+08:00","level":"INFO","msg":"sender: started","stream_id":"iunrz6lf"} +{"time":"2025-08-13T14:29:43.486140148+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062942Z&X-Goog-Expires=86399&X-Goog-Signature=2f0d6f5f9c5c89e54bc0d24076ee3ada852cc3d8927e4f1458c329da822f5b1ef992e61fb14afbb502784e633001aee460c1e6b41f9c08d0e95ed7c3672da7dfb8599fa9e963604890d5823db21bcac112368aede49bb340e45f7a2ae05b73c11e58fca0585abbae4ca6c6fdbb6bd5124551ff9eb94ce975e2970f4c330af812c20df1aadddbe0cdd69b98ff8c6f0ac38b5c63aea339dfe9e0d708634f6c82a7a9ec197f7ed2fa1f4d086a7249b559e2bee1d7bfa5f06c9539b6d015668b1cd65a2ae802066366a3d43e66a4ebf891ba39e014824efb217751f0b3ef33eca0abb2fd38b72e897d2d73a1f1037f5bbbe0cce63731dad67f458704310ed90883c7&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:33632->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062942Z&X-Goog-Expires=86399&X-Goog-Signature=2f0d6f5f9c5c89e54bc0d24076ee3ada852cc3d8927e4f1458c329da822f5b1ef992e61fb14afbb502784e633001aee460c1e6b41f9c08d0e95ed7c3672da7dfb8599fa9e963604890d5823db21bcac112368aede49bb340e45f7a2ae05b73c11e58fca0585abbae4ca6c6fdbb6bd5124551ff9eb94ce975e2970f4c330af812c20df1aadddbe0cdd69b98ff8c6f0ac38b5c63aea339dfe9e0d708634f6c82a7a9ec197f7ed2fa1f4d086a7249b559e2bee1d7bfa5f06c9539b6d015668b1cd65a2ae802066366a3d43e66a4ebf891ba39e014824efb217751f0b3ef33eca0abb2fd38b72e897d2d73a1f1037f5bbbe0cce63731dad67f458704310ed90883c7&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:29:45.374722442+08:00","level":"INFO","msg":"stream: closing","id":"iunrz6lf"} +{"time":"2025-08-13T14:29:46.73264407+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062945Z&X-Goog-Expires=86399&X-Goog-Signature=a0cd43d546e7b1629fdd510e714b1e805fc6c949c3770f98462a4e2c829718818c94b8c4400e4ebaef7d7caf1f5a34e7f6e502cc0fb215e01033fc4e6f2fbcf335647fee8588a8372a304a03cd2d525bf8e70f9f4d7d14586a0d6e85b4b44c8b9fdd3e25d020d58267b4b70ec0b3e74c0c7d85dcca6c1e8f3f324f9d608e027c2093a116416b83e73a60665274a11408d4bd1069c15f141a588015f412feb03539a645be245abef074df01dfb9a3cf27621fd74a8d9f4c474e725f6f29a96a41b5ec75bf01c51e94ce383204248e186f5f72006f8993831019673bc3b27512dc8e255254db1f537d1d2ce8d128bd7f6549a0e89574613d96677b8079533bf978&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:51430->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062945Z&X-Goog-Expires=86399&X-Goog-Signature=a0cd43d546e7b1629fdd510e714b1e805fc6c949c3770f98462a4e2c829718818c94b8c4400e4ebaef7d7caf1f5a34e7f6e502cc0fb215e01033fc4e6f2fbcf335647fee8588a8372a304a03cd2d525bf8e70f9f4d7d14586a0d6e85b4b44c8b9fdd3e25d020d58267b4b70ec0b3e74c0c7d85dcca6c1e8f3f324f9d608e027c2093a116416b83e73a60665274a11408d4bd1069c15f141a588015f412feb03539a645be245abef074df01dfb9a3cf27621fd74a8d9f4c474e725f6f29a96a41b5ec75bf01c51e94ce383204248e186f5f72006f8993831019673bc3b27512dc8e255254db1f537d1d2ce8d128bd7f6549a0e89574613d96677b8079533bf978&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:29:49.431974131+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062946Z&X-Goog-Expires=86399&X-Goog-Signature=a9fb26867862d7a872be0fe3943b1c45fa69319d509095ad8711be6c25366fe74232e8c8d4206f570068fde288c489ba063fbcd62bc5b95346a8b7d4944d3bc9b7851a92d2cb31647084284af54cb078ea6ead75199b810ef4ed2c49532395be0fb8446e76e25d84440cad7825f70805891e6caf9ffab8dff220311a0d61928d3568c8be34b5223bb49f33d70f8cd9cb98e38c7198f94666243187d53e91a6d9f9c587b073362486fc9d2e0c32a7b254b874b26dd04e48fb1d5f3363547f2fbd353f23ba7430f8862bed5c51094227543427c5903d7b91f8b82fb1430be2359ab81a144263d199050eb36464deb35b28fd386bb04bd9350e7f4434ca7d1b4691&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:45610->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062946Z&X-Goog-Expires=86399&X-Goog-Signature=a9fb26867862d7a872be0fe3943b1c45fa69319d509095ad8711be6c25366fe74232e8c8d4206f570068fde288c489ba063fbcd62bc5b95346a8b7d4944d3bc9b7851a92d2cb31647084284af54cb078ea6ead75199b810ef4ed2c49532395be0fb8446e76e25d84440cad7825f70805891e6caf9ffab8dff220311a0d61928d3568c8be34b5223bb49f33d70f8cd9cb98e38c7198f94666243187d53e91a6d9f9c587b073362486fc9d2e0c32a7b254b874b26dd04e48fb1d5f3363547f2fbd353f23ba7430f8862bed5c51094227543427c5903d7b91f8b82fb1430be2359ab81a144263d199050eb36464deb35b28fd386bb04bd9350e7f4434ca7d1b4691&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:31:55.704727055+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062945Z&X-Goog-Expires=86399&X-Goog-Signature=08133535cc069bd54e37428d9c3e8fa600c79f990b330e9d93f9fb8eb2743163bc21ecf8894027e63c0ccefb0b8670ea098df11d3a05317c0ffdce5aa902b1bd0af7594dcebb433956dd83300e89a0936c7bab3374dc48bbb6acd43d7979e6ba0d64c714b87f20dc93305234af3bf738ec194b037406485c03f820fae18c2afde4afe64d084fd9fa9031f8a1a0b035d3857e792a0b86b87876d672348967876f1c64aa1e7399ac602d06041399a232264770f2773e3f35ce1e8a09d5cdd2c551be22c13facacd1e803de3a360bc4d7f03b43fc9d3349d7e917786769b3e0d16f959ea752b0f7c24ef872d36447e7aa1de1d09d9f8165f51eb7dd8ae557c3f7c7&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:47612->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062945Z&X-Goog-Expires=86399&X-Goog-Signature=08133535cc069bd54e37428d9c3e8fa600c79f990b330e9d93f9fb8eb2743163bc21ecf8894027e63c0ccefb0b8670ea098df11d3a05317c0ffdce5aa902b1bd0af7594dcebb433956dd83300e89a0936c7bab3374dc48bbb6acd43d7979e6ba0d64c714b87f20dc93305234af3bf738ec194b037406485c03f820fae18c2afde4afe64d084fd9fa9031f8a1a0b035d3857e792a0b86b87876d672348967876f1c64aa1e7399ac602d06041399a232264770f2773e3f35ce1e8a09d5cdd2c551be22c13facacd1e803de3a360bc4d7f03b43fc9d3349d7e917786769b3e0d16f959ea752b0f7c24ef872d36447e7aa1de1d09d9f8165f51eb7dd8ae557c3f7c7&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:31:58.201731984+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062945Z&X-Goog-Expires=86399&X-Goog-Signature=08133535cc069bd54e37428d9c3e8fa600c79f990b330e9d93f9fb8eb2743163bc21ecf8894027e63c0ccefb0b8670ea098df11d3a05317c0ffdce5aa902b1bd0af7594dcebb433956dd83300e89a0936c7bab3374dc48bbb6acd43d7979e6ba0d64c714b87f20dc93305234af3bf738ec194b037406485c03f820fae18c2afde4afe64d084fd9fa9031f8a1a0b035d3857e792a0b86b87876d672348967876f1c64aa1e7399ac602d06041399a232264770f2773e3f35ce1e8a09d5cdd2c551be22c13facacd1e803de3a360bc4d7f03b43fc9d3349d7e917786769b3e0d16f959ea752b0f7c24ef872d36447e7aa1de1d09d9f8165f51eb7dd8ae557c3f7c7&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:47248->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062945Z&X-Goog-Expires=86399&X-Goog-Signature=08133535cc069bd54e37428d9c3e8fa600c79f990b330e9d93f9fb8eb2743163bc21ecf8894027e63c0ccefb0b8670ea098df11d3a05317c0ffdce5aa902b1bd0af7594dcebb433956dd83300e89a0936c7bab3374dc48bbb6acd43d7979e6ba0d64c714b87f20dc93305234af3bf738ec194b037406485c03f820fae18c2afde4afe64d084fd9fa9031f8a1a0b035d3857e792a0b86b87876d672348967876f1c64aa1e7399ac602d06041399a232264770f2773e3f35ce1e8a09d5cdd2c551be22c13facacd1e803de3a360bc4d7f03b43fc9d3349d7e917786769b3e0d16f959ea752b0f7c24ef872d36447e7aa1de1d09d9f8165f51eb7dd8ae557c3f7c7&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:34:06.797902395+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062942Z&X-Goog-Expires=86399&X-Goog-Signature=2f0d6f5f9c5c89e54bc0d24076ee3ada852cc3d8927e4f1458c329da822f5b1ef992e61fb14afbb502784e633001aee460c1e6b41f9c08d0e95ed7c3672da7dfb8599fa9e963604890d5823db21bcac112368aede49bb340e45f7a2ae05b73c11e58fca0585abbae4ca6c6fdbb6bd5124551ff9eb94ce975e2970f4c330af812c20df1aadddbe0cdd69b98ff8c6f0ac38b5c63aea339dfe9e0d708634f6c82a7a9ec197f7ed2fa1f4d086a7249b559e2bee1d7bfa5f06c9539b6d015668b1cd65a2ae802066366a3d43e66a4ebf891ba39e014824efb217751f0b3ef33eca0abb2fd38b72e897d2d73a1f1037f5bbbe0cce63731dad67f458704310ed90883c7&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:50710->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062942Z&X-Goog-Expires=86399&X-Goog-Signature=2f0d6f5f9c5c89e54bc0d24076ee3ada852cc3d8927e4f1458c329da822f5b1ef992e61fb14afbb502784e633001aee460c1e6b41f9c08d0e95ed7c3672da7dfb8599fa9e963604890d5823db21bcac112368aede49bb340e45f7a2ae05b73c11e58fca0585abbae4ca6c6fdbb6bd5124551ff9eb94ce975e2970f4c330af812c20df1aadddbe0cdd69b98ff8c6f0ac38b5c63aea339dfe9e0d708634f6c82a7a9ec197f7ed2fa1f4d086a7249b559e2bee1d7bfa5f06c9539b6d015668b1cd65a2ae802066366a3d43e66a4ebf891ba39e014824efb217751f0b3ef33eca0abb2fd38b72e897d2d73a1f1037f5bbbe0cce63731dad67f458704310ed90883c7&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:34:10.844728612+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062945Z&X-Goog-Expires=86399&X-Goog-Signature=a0cd43d546e7b1629fdd510e714b1e805fc6c949c3770f98462a4e2c829718818c94b8c4400e4ebaef7d7caf1f5a34e7f6e502cc0fb215e01033fc4e6f2fbcf335647fee8588a8372a304a03cd2d525bf8e70f9f4d7d14586a0d6e85b4b44c8b9fdd3e25d020d58267b4b70ec0b3e74c0c7d85dcca6c1e8f3f324f9d608e027c2093a116416b83e73a60665274a11408d4bd1069c15f141a588015f412feb03539a645be245abef074df01dfb9a3cf27621fd74a8d9f4c474e725f6f29a96a41b5ec75bf01c51e94ce383204248e186f5f72006f8993831019673bc3b27512dc8e255254db1f537d1d2ce8d128bd7f6549a0e89574613d96677b8079533bf978&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:39586->142.250.73.155:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062945Z&X-Goog-Expires=86399&X-Goog-Signature=a0cd43d546e7b1629fdd510e714b1e805fc6c949c3770f98462a4e2c829718818c94b8c4400e4ebaef7d7caf1f5a34e7f6e502cc0fb215e01033fc4e6f2fbcf335647fee8588a8372a304a03cd2d525bf8e70f9f4d7d14586a0d6e85b4b44c8b9fdd3e25d020d58267b4b70ec0b3e74c0c7d85dcca6c1e8f3f324f9d608e027c2093a116416b83e73a60665274a11408d4bd1069c15f141a588015f412feb03539a645be245abef074df01dfb9a3cf27621fd74a8d9f4c474e725f6f29a96a41b5ec75bf01c51e94ce383204248e186f5f72006f8993831019673bc3b27512dc8e255254db1f537d1d2ce8d128bd7f6549a0e89574613d96677b8079533bf978&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:34:11.231196271+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062942Z&X-Goog-Expires=86399&X-Goog-Signature=2f0d6f5f9c5c89e54bc0d24076ee3ada852cc3d8927e4f1458c329da822f5b1ef992e61fb14afbb502784e633001aee460c1e6b41f9c08d0e95ed7c3672da7dfb8599fa9e963604890d5823db21bcac112368aede49bb340e45f7a2ae05b73c11e58fca0585abbae4ca6c6fdbb6bd5124551ff9eb94ce975e2970f4c330af812c20df1aadddbe0cdd69b98ff8c6f0ac38b5c63aea339dfe9e0d708634f6c82a7a9ec197f7ed2fa1f4d086a7249b559e2bee1d7bfa5f06c9539b6d015668b1cd65a2ae802066366a3d43e66a4ebf891ba39e014824efb217751f0b3ef33eca0abb2fd38b72e897d2d73a1f1037f5bbbe0cce63731dad67f458704310ed90883c7&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:40664->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062942Z&X-Goog-Expires=86399&X-Goog-Signature=2f0d6f5f9c5c89e54bc0d24076ee3ada852cc3d8927e4f1458c329da822f5b1ef992e61fb14afbb502784e633001aee460c1e6b41f9c08d0e95ed7c3672da7dfb8599fa9e963604890d5823db21bcac112368aede49bb340e45f7a2ae05b73c11e58fca0585abbae4ca6c6fdbb6bd5124551ff9eb94ce975e2970f4c330af812c20df1aadddbe0cdd69b98ff8c6f0ac38b5c63aea339dfe9e0d708634f6c82a7a9ec197f7ed2fa1f4d086a7249b559e2bee1d7bfa5f06c9539b6d015668b1cd65a2ae802066366a3d43e66a4ebf891ba39e014824efb217751f0b3ef33eca0abb2fd38b72e897d2d73a1f1037f5bbbe0cce63731dad67f458704310ed90883c7&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:34:14.997560565+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062946Z&X-Goog-Expires=86399&X-Goog-Signature=a9fb26867862d7a872be0fe3943b1c45fa69319d509095ad8711be6c25366fe74232e8c8d4206f570068fde288c489ba063fbcd62bc5b95346a8b7d4944d3bc9b7851a92d2cb31647084284af54cb078ea6ead75199b810ef4ed2c49532395be0fb8446e76e25d84440cad7825f70805891e6caf9ffab8dff220311a0d61928d3568c8be34b5223bb49f33d70f8cd9cb98e38c7198f94666243187d53e91a6d9f9c587b073362486fc9d2e0c32a7b254b874b26dd04e48fb1d5f3363547f2fbd353f23ba7430f8862bed5c51094227543427c5903d7b91f8b82fb1430be2359ab81a144263d199050eb36464deb35b28fd386bb04bd9350e7f4434ca7d1b4691&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:49688->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062946Z&X-Goog-Expires=86399&X-Goog-Signature=a9fb26867862d7a872be0fe3943b1c45fa69319d509095ad8711be6c25366fe74232e8c8d4206f570068fde288c489ba063fbcd62bc5b95346a8b7d4944d3bc9b7851a92d2cb31647084284af54cb078ea6ead75199b810ef4ed2c49532395be0fb8446e76e25d84440cad7825f70805891e6caf9ffab8dff220311a0d61928d3568c8be34b5223bb49f33d70f8cd9cb98e38c7198f94666243187d53e91a6d9f9c587b073362486fc9d2e0c32a7b254b874b26dd04e48fb1d5f3363547f2fbd353f23ba7430f8862bed5c51094227543427c5903d7b91f8b82fb1430be2359ab81a144263d199050eb36464deb35b28fd386bb04bd9350e7f4434ca7d1b4691&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:34:15.725410473+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062945Z&X-Goog-Expires=86399&X-Goog-Signature=a0cd43d546e7b1629fdd510e714b1e805fc6c949c3770f98462a4e2c829718818c94b8c4400e4ebaef7d7caf1f5a34e7f6e502cc0fb215e01033fc4e6f2fbcf335647fee8588a8372a304a03cd2d525bf8e70f9f4d7d14586a0d6e85b4b44c8b9fdd3e25d020d58267b4b70ec0b3e74c0c7d85dcca6c1e8f3f324f9d608e027c2093a116416b83e73a60665274a11408d4bd1069c15f141a588015f412feb03539a645be245abef074df01dfb9a3cf27621fd74a8d9f4c474e725f6f29a96a41b5ec75bf01c51e94ce383204248e186f5f72006f8993831019673bc3b27512dc8e255254db1f537d1d2ce8d128bd7f6549a0e89574613d96677b8079533bf978&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:40676->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062945Z&X-Goog-Expires=86399&X-Goog-Signature=a0cd43d546e7b1629fdd510e714b1e805fc6c949c3770f98462a4e2c829718818c94b8c4400e4ebaef7d7caf1f5a34e7f6e502cc0fb215e01033fc4e6f2fbcf335647fee8588a8372a304a03cd2d525bf8e70f9f4d7d14586a0d6e85b4b44c8b9fdd3e25d020d58267b4b70ec0b3e74c0c7d85dcca6c1e8f3f324f9d608e027c2093a116416b83e73a60665274a11408d4bd1069c15f141a588015f412feb03539a645be245abef074df01dfb9a3cf27621fd74a8d9f4c474e725f6f29a96a41b5ec75bf01c51e94ce383204248e186f5f72006f8993831019673bc3b27512dc8e255254db1f537d1d2ce8d128bd7f6549a0e89574613d96677b8079533bf978&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:34:21.375377934+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062942Z&X-Goog-Expires=86399&X-Goog-Signature=2f0d6f5f9c5c89e54bc0d24076ee3ada852cc3d8927e4f1458c329da822f5b1ef992e61fb14afbb502784e633001aee460c1e6b41f9c08d0e95ed7c3672da7dfb8599fa9e963604890d5823db21bcac112368aede49bb340e45f7a2ae05b73c11e58fca0585abbae4ca6c6fdbb6bd5124551ff9eb94ce975e2970f4c330af812c20df1aadddbe0cdd69b98ff8c6f0ac38b5c63aea339dfe9e0d708634f6c82a7a9ec197f7ed2fa1f4d086a7249b559e2bee1d7bfa5f06c9539b6d015668b1cd65a2ae802066366a3d43e66a4ebf891ba39e014824efb217751f0b3ef33eca0abb2fd38b72e897d2d73a1f1037f5bbbe0cce63731dad67f458704310ed90883c7&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:34200->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062942Z&X-Goog-Expires=86399&X-Goog-Signature=2f0d6f5f9c5c89e54bc0d24076ee3ada852cc3d8927e4f1458c329da822f5b1ef992e61fb14afbb502784e633001aee460c1e6b41f9c08d0e95ed7c3672da7dfb8599fa9e963604890d5823db21bcac112368aede49bb340e45f7a2ae05b73c11e58fca0585abbae4ca6c6fdbb6bd5124551ff9eb94ce975e2970f4c330af812c20df1aadddbe0cdd69b98ff8c6f0ac38b5c63aea339dfe9e0d708634f6c82a7a9ec197f7ed2fa1f4d086a7249b559e2bee1d7bfa5f06c9539b6d015668b1cd65a2ae802066366a3d43e66a4ebf891ba39e014824efb217751f0b3ef33eca0abb2fd38b72e897d2d73a1f1037f5bbbe0cce63731dad67f458704310ed90883c7&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:34:25.610475761+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062945Z&X-Goog-Expires=86399&X-Goog-Signature=a0cd43d546e7b1629fdd510e714b1e805fc6c949c3770f98462a4e2c829718818c94b8c4400e4ebaef7d7caf1f5a34e7f6e502cc0fb215e01033fc4e6f2fbcf335647fee8588a8372a304a03cd2d525bf8e70f9f4d7d14586a0d6e85b4b44c8b9fdd3e25d020d58267b4b70ec0b3e74c0c7d85dcca6c1e8f3f324f9d608e027c2093a116416b83e73a60665274a11408d4bd1069c15f141a588015f412feb03539a645be245abef074df01dfb9a3cf27621fd74a8d9f4c474e725f6f29a96a41b5ec75bf01c51e94ce383204248e186f5f72006f8993831019673bc3b27512dc8e255254db1f537d1d2ce8d128bd7f6549a0e89574613d96677b8079533bf978&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:34202->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062945Z&X-Goog-Expires=86399&X-Goog-Signature=a0cd43d546e7b1629fdd510e714b1e805fc6c949c3770f98462a4e2c829718818c94b8c4400e4ebaef7d7caf1f5a34e7f6e502cc0fb215e01033fc4e6f2fbcf335647fee8588a8372a304a03cd2d525bf8e70f9f4d7d14586a0d6e85b4b44c8b9fdd3e25d020d58267b4b70ec0b3e74c0c7d85dcca6c1e8f3f324f9d608e027c2093a116416b83e73a60665274a11408d4bd1069c15f141a588015f412feb03539a645be245abef074df01dfb9a3cf27621fd74a8d9f4c474e725f6f29a96a41b5ec75bf01c51e94ce383204248e186f5f72006f8993831019673bc3b27512dc8e255254db1f537d1d2ce8d128bd7f6549a0e89574613d96677b8079533bf978&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:34:44.404398165+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062945Z&X-Goog-Expires=86399&X-Goog-Signature=a0cd43d546e7b1629fdd510e714b1e805fc6c949c3770f98462a4e2c829718818c94b8c4400e4ebaef7d7caf1f5a34e7f6e502cc0fb215e01033fc4e6f2fbcf335647fee8588a8372a304a03cd2d525bf8e70f9f4d7d14586a0d6e85b4b44c8b9fdd3e25d020d58267b4b70ec0b3e74c0c7d85dcca6c1e8f3f324f9d608e027c2093a116416b83e73a60665274a11408d4bd1069c15f141a588015f412feb03539a645be245abef074df01dfb9a3cf27621fd74a8d9f4c474e725f6f29a96a41b5ec75bf01c51e94ce383204248e186f5f72006f8993831019673bc3b27512dc8e255254db1f537d1d2ce8d128bd7f6549a0e89574613d96677b8079533bf978&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:45466->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062945Z&X-Goog-Expires=86399&X-Goog-Signature=a0cd43d546e7b1629fdd510e714b1e805fc6c949c3770f98462a4e2c829718818c94b8c4400e4ebaef7d7caf1f5a34e7f6e502cc0fb215e01033fc4e6f2fbcf335647fee8588a8372a304a03cd2d525bf8e70f9f4d7d14586a0d6e85b4b44c8b9fdd3e25d020d58267b4b70ec0b3e74c0c7d85dcca6c1e8f3f324f9d608e027c2093a116416b83e73a60665274a11408d4bd1069c15f141a588015f412feb03539a645be245abef074df01dfb9a3cf27621fd74a8d9f4c474e725f6f29a96a41b5ec75bf01c51e94ce383204248e186f5f72006f8993831019673bc3b27512dc8e255254db1f537d1d2ce8d128bd7f6549a0e89574613d96677b8079533bf978&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:36:17.803692253+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062942Z&X-Goog-Expires=86399&X-Goog-Signature=0e1836035673f0fd54fb10b2125f785db2fa1622625c3cffb63136b19a3ad03b454542d5dd49df7faf9cd79b6e64bd767d87c72f8e4014a357770b9ad927ce9c9c26b6cd1e950e9c06265285e664b8e2d3e977f5d53377a808e46e0d979c441492a682f62274ccc9dbb452c5f131f0e19ea29a0c0b65dc6b3397e984e02c0064bce8aa2b1b53ac955e0eeb3d46b19538f4d3cf434d9e67b7f41f222f4299131c63411d9fec4b89966083e067dd522453cfc6ea3ca1fa842fceaf4fec57515c090b328e1c88d85237be6e8412cd41ff83017e917b1d00f5412c9990b8183edfd3ceaaefc4b58599935d6163ad604b023e56f100d505ccca012db8c7149b306910&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:39866->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062942Z&X-Goog-Expires=86399&X-Goog-Signature=0e1836035673f0fd54fb10b2125f785db2fa1622625c3cffb63136b19a3ad03b454542d5dd49df7faf9cd79b6e64bd767d87c72f8e4014a357770b9ad927ce9c9c26b6cd1e950e9c06265285e664b8e2d3e977f5d53377a808e46e0d979c441492a682f62274ccc9dbb452c5f131f0e19ea29a0c0b65dc6b3397e984e02c0064bce8aa2b1b53ac955e0eeb3d46b19538f4d3cf434d9e67b7f41f222f4299131c63411d9fec4b89966083e067dd522453cfc6ea3ca1fa842fceaf4fec57515c090b328e1c88d85237be6e8412cd41ff83017e917b1d00f5412c9990b8183edfd3ceaaefc4b58599935d6163ad604b023e56f100d505ccca012db8c7149b306910&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:36:20.104009313+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062942Z&X-Goog-Expires=86399&X-Goog-Signature=0e1836035673f0fd54fb10b2125f785db2fa1622625c3cffb63136b19a3ad03b454542d5dd49df7faf9cd79b6e64bd767d87c72f8e4014a357770b9ad927ce9c9c26b6cd1e950e9c06265285e664b8e2d3e977f5d53377a808e46e0d979c441492a682f62274ccc9dbb452c5f131f0e19ea29a0c0b65dc6b3397e984e02c0064bce8aa2b1b53ac955e0eeb3d46b19538f4d3cf434d9e67b7f41f222f4299131c63411d9fec4b89966083e067dd522453cfc6ea3ca1fa842fceaf4fec57515c090b328e1c88d85237be6e8412cd41ff83017e917b1d00f5412c9990b8183edfd3ceaaefc4b58599935d6163ad604b023e56f100d505ccca012db8c7149b306910&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:47998->142.250.73.155:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062942Z&X-Goog-Expires=86399&X-Goog-Signature=0e1836035673f0fd54fb10b2125f785db2fa1622625c3cffb63136b19a3ad03b454542d5dd49df7faf9cd79b6e64bd767d87c72f8e4014a357770b9ad927ce9c9c26b6cd1e950e9c06265285e664b8e2d3e977f5d53377a808e46e0d979c441492a682f62274ccc9dbb452c5f131f0e19ea29a0c0b65dc6b3397e984e02c0064bce8aa2b1b53ac955e0eeb3d46b19538f4d3cf434d9e67b7f41f222f4299131c63411d9fec4b89966083e067dd522453cfc6ea3ca1fa842fceaf4fec57515c090b328e1c88d85237be6e8412cd41ff83017e917b1d00f5412c9990b8183edfd3ceaaefc4b58599935d6163ad604b023e56f100d505ccca012db8c7149b306910&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:36:24.787732621+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062942Z&X-Goog-Expires=86399&X-Goog-Signature=0e1836035673f0fd54fb10b2125f785db2fa1622625c3cffb63136b19a3ad03b454542d5dd49df7faf9cd79b6e64bd767d87c72f8e4014a357770b9ad927ce9c9c26b6cd1e950e9c06265285e664b8e2d3e977f5d53377a808e46e0d979c441492a682f62274ccc9dbb452c5f131f0e19ea29a0c0b65dc6b3397e984e02c0064bce8aa2b1b53ac955e0eeb3d46b19538f4d3cf434d9e67b7f41f222f4299131c63411d9fec4b89966083e067dd522453cfc6ea3ca1fa842fceaf4fec57515c090b328e1c88d85237be6e8412cd41ff83017e917b1d00f5412c9990b8183edfd3ceaaefc4b58599935d6163ad604b023e56f100d505ccca012db8c7149b306910&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:59700->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062942Z&X-Goog-Expires=86399&X-Goog-Signature=0e1836035673f0fd54fb10b2125f785db2fa1622625c3cffb63136b19a3ad03b454542d5dd49df7faf9cd79b6e64bd767d87c72f8e4014a357770b9ad927ce9c9c26b6cd1e950e9c06265285e664b8e2d3e977f5d53377a808e46e0d979c441492a682f62274ccc9dbb452c5f131f0e19ea29a0c0b65dc6b3397e984e02c0064bce8aa2b1b53ac955e0eeb3d46b19538f4d3cf434d9e67b7f41f222f4299131c63411d9fec4b89966083e067dd522453cfc6ea3ca1fa842fceaf4fec57515c090b328e1c88d85237be6e8412cd41ff83017e917b1d00f5412c9990b8183edfd3ceaaefc4b58599935d6163ad604b023e56f100d505ccca012db8c7149b306910&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:36:26.015427949+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062945Z&X-Goog-Expires=86399&X-Goog-Signature=08133535cc069bd54e37428d9c3e8fa600c79f990b330e9d93f9fb8eb2743163bc21ecf8894027e63c0ccefb0b8670ea098df11d3a05317c0ffdce5aa902b1bd0af7594dcebb433956dd83300e89a0936c7bab3374dc48bbb6acd43d7979e6ba0d64c714b87f20dc93305234af3bf738ec194b037406485c03f820fae18c2afde4afe64d084fd9fa9031f8a1a0b035d3857e792a0b86b87876d672348967876f1c64aa1e7399ac602d06041399a232264770f2773e3f35ce1e8a09d5cdd2c551be22c13facacd1e803de3a360bc4d7f03b43fc9d3349d7e917786769b3e0d16f959ea752b0f7c24ef872d36447e7aa1de1d09d9f8165f51eb7dd8ae557c3f7c7&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:59716->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062945Z&X-Goog-Expires=86399&X-Goog-Signature=08133535cc069bd54e37428d9c3e8fa600c79f990b330e9d93f9fb8eb2743163bc21ecf8894027e63c0ccefb0b8670ea098df11d3a05317c0ffdce5aa902b1bd0af7594dcebb433956dd83300e89a0936c7bab3374dc48bbb6acd43d7979e6ba0d64c714b87f20dc93305234af3bf738ec194b037406485c03f820fae18c2afde4afe64d084fd9fa9031f8a1a0b035d3857e792a0b86b87876d672348967876f1c64aa1e7399ac602d06041399a232264770f2773e3f35ce1e8a09d5cdd2c551be22c13facacd1e803de3a360bc4d7f03b43fc9d3349d7e917786769b3e0d16f959ea752b0f7c24ef872d36447e7aa1de1d09d9f8165f51eb7dd8ae557c3f7c7&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:38:41.235384044+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062946Z&X-Goog-Expires=86399&X-Goog-Signature=a9fb26867862d7a872be0fe3943b1c45fa69319d509095ad8711be6c25366fe74232e8c8d4206f570068fde288c489ba063fbcd62bc5b95346a8b7d4944d3bc9b7851a92d2cb31647084284af54cb078ea6ead75199b810ef4ed2c49532395be0fb8446e76e25d84440cad7825f70805891e6caf9ffab8dff220311a0d61928d3568c8be34b5223bb49f33d70f8cd9cb98e38c7198f94666243187d53e91a6d9f9c587b073362486fc9d2e0c32a7b254b874b26dd04e48fb1d5f3363547f2fbd353f23ba7430f8862bed5c51094227543427c5903d7b91f8b82fb1430be2359ab81a144263d199050eb36464deb35b28fd386bb04bd9350e7f4434ca7d1b4691&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:58594->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062946Z&X-Goog-Expires=86399&X-Goog-Signature=a9fb26867862d7a872be0fe3943b1c45fa69319d509095ad8711be6c25366fe74232e8c8d4206f570068fde288c489ba063fbcd62bc5b95346a8b7d4944d3bc9b7851a92d2cb31647084284af54cb078ea6ead75199b810ef4ed2c49532395be0fb8446e76e25d84440cad7825f70805891e6caf9ffab8dff220311a0d61928d3568c8be34b5223bb49f33d70f8cd9cb98e38c7198f94666243187d53e91a6d9f9c587b073362486fc9d2e0c32a7b254b874b26dd04e48fb1d5f3363547f2fbd353f23ba7430f8862bed5c51094227543427c5903d7b91f8b82fb1430be2359ab81a144263d199050eb36464deb35b28fd386bb04bd9350e7f4434ca7d1b4691&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:38:45.314577369+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062945Z&X-Goog-Expires=86399&X-Goog-Signature=08133535cc069bd54e37428d9c3e8fa600c79f990b330e9d93f9fb8eb2743163bc21ecf8894027e63c0ccefb0b8670ea098df11d3a05317c0ffdce5aa902b1bd0af7594dcebb433956dd83300e89a0936c7bab3374dc48bbb6acd43d7979e6ba0d64c714b87f20dc93305234af3bf738ec194b037406485c03f820fae18c2afde4afe64d084fd9fa9031f8a1a0b035d3857e792a0b86b87876d672348967876f1c64aa1e7399ac602d06041399a232264770f2773e3f35ce1e8a09d5cdd2c551be22c13facacd1e803de3a360bc4d7f03b43fc9d3349d7e917786769b3e0d16f959ea752b0f7c24ef872d36447e7aa1de1d09d9f8165f51eb7dd8ae557c3f7c7&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:58600->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062945Z&X-Goog-Expires=86399&X-Goog-Signature=08133535cc069bd54e37428d9c3e8fa600c79f990b330e9d93f9fb8eb2743163bc21ecf8894027e63c0ccefb0b8670ea098df11d3a05317c0ffdce5aa902b1bd0af7594dcebb433956dd83300e89a0936c7bab3374dc48bbb6acd43d7979e6ba0d64c714b87f20dc93305234af3bf738ec194b037406485c03f820fae18c2afde4afe64d084fd9fa9031f8a1a0b035d3857e792a0b86b87876d672348967876f1c64aa1e7399ac602d06041399a232264770f2773e3f35ce1e8a09d5cdd2c551be22c13facacd1e803de3a360bc4d7f03b43fc9d3349d7e917786769b3e0d16f959ea752b0f7c24ef872d36447e7aa1de1d09d9f8165f51eb7dd8ae557c3f7c7&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:38:50.634715669+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062946Z&X-Goog-Expires=86399&X-Goog-Signature=a9fb26867862d7a872be0fe3943b1c45fa69319d509095ad8711be6c25366fe74232e8c8d4206f570068fde288c489ba063fbcd62bc5b95346a8b7d4944d3bc9b7851a92d2cb31647084284af54cb078ea6ead75199b810ef4ed2c49532395be0fb8446e76e25d84440cad7825f70805891e6caf9ffab8dff220311a0d61928d3568c8be34b5223bb49f33d70f8cd9cb98e38c7198f94666243187d53e91a6d9f9c587b073362486fc9d2e0c32a7b254b874b26dd04e48fb1d5f3363547f2fbd353f23ba7430f8862bed5c51094227543427c5903d7b91f8b82fb1430be2359ab81a144263d199050eb36464deb35b28fd386bb04bd9350e7f4434ca7d1b4691&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:43628->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062946Z&X-Goog-Expires=86399&X-Goog-Signature=a9fb26867862d7a872be0fe3943b1c45fa69319d509095ad8711be6c25366fe74232e8c8d4206f570068fde288c489ba063fbcd62bc5b95346a8b7d4944d3bc9b7851a92d2cb31647084284af54cb078ea6ead75199b810ef4ed2c49532395be0fb8446e76e25d84440cad7825f70805891e6caf9ffab8dff220311a0d61928d3568c8be34b5223bb49f33d70f8cd9cb98e38c7198f94666243187d53e91a6d9f9c587b073362486fc9d2e0c32a7b254b874b26dd04e48fb1d5f3363547f2fbd353f23ba7430f8862bed5c51094227543427c5903d7b91f8b82fb1430be2359ab81a144263d199050eb36464deb35b28fd386bb04bd9350e7f4434ca7d1b4691&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:39:38.564158425+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062945Z&X-Goog-Expires=86399&X-Goog-Signature=a0cd43d546e7b1629fdd510e714b1e805fc6c949c3770f98462a4e2c829718818c94b8c4400e4ebaef7d7caf1f5a34e7f6e502cc0fb215e01033fc4e6f2fbcf335647fee8588a8372a304a03cd2d525bf8e70f9f4d7d14586a0d6e85b4b44c8b9fdd3e25d020d58267b4b70ec0b3e74c0c7d85dcca6c1e8f3f324f9d608e027c2093a116416b83e73a60665274a11408d4bd1069c15f141a588015f412feb03539a645be245abef074df01dfb9a3cf27621fd74a8d9f4c474e725f6f29a96a41b5ec75bf01c51e94ce383204248e186f5f72006f8993831019673bc3b27512dc8e255254db1f537d1d2ce8d128bd7f6549a0e89574613d96677b8079533bf978&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:34124->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062945Z&X-Goog-Expires=86399&X-Goog-Signature=a0cd43d546e7b1629fdd510e714b1e805fc6c949c3770f98462a4e2c829718818c94b8c4400e4ebaef7d7caf1f5a34e7f6e502cc0fb215e01033fc4e6f2fbcf335647fee8588a8372a304a03cd2d525bf8e70f9f4d7d14586a0d6e85b4b44c8b9fdd3e25d020d58267b4b70ec0b3e74c0c7d85dcca6c1e8f3f324f9d608e027c2093a116416b83e73a60665274a11408d4bd1069c15f141a588015f412feb03539a645be245abef074df01dfb9a3cf27621fd74a8d9f4c474e725f6f29a96a41b5ec75bf01c51e94ce383204248e186f5f72006f8993831019673bc3b27512dc8e255254db1f537d1d2ce8d128bd7f6549a0e89574613d96677b8079533bf978&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:40:56.323112266+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062942Z&X-Goog-Expires=86399&X-Goog-Signature=0e1836035673f0fd54fb10b2125f785db2fa1622625c3cffb63136b19a3ad03b454542d5dd49df7faf9cd79b6e64bd767d87c72f8e4014a357770b9ad927ce9c9c26b6cd1e950e9c06265285e664b8e2d3e977f5d53377a808e46e0d979c441492a682f62274ccc9dbb452c5f131f0e19ea29a0c0b65dc6b3397e984e02c0064bce8aa2b1b53ac955e0eeb3d46b19538f4d3cf434d9e67b7f41f222f4299131c63411d9fec4b89966083e067dd522453cfc6ea3ca1fa842fceaf4fec57515c090b328e1c88d85237be6e8412cd41ff83017e917b1d00f5412c9990b8183edfd3ceaaefc4b58599935d6163ad604b023e56f100d505ccca012db8c7149b306910&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:39600->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062942Z&X-Goog-Expires=86399&X-Goog-Signature=0e1836035673f0fd54fb10b2125f785db2fa1622625c3cffb63136b19a3ad03b454542d5dd49df7faf9cd79b6e64bd767d87c72f8e4014a357770b9ad927ce9c9c26b6cd1e950e9c06265285e664b8e2d3e977f5d53377a808e46e0d979c441492a682f62274ccc9dbb452c5f131f0e19ea29a0c0b65dc6b3397e984e02c0064bce8aa2b1b53ac955e0eeb3d46b19538f4d3cf434d9e67b7f41f222f4299131c63411d9fec4b89966083e067dd522453cfc6ea3ca1fa842fceaf4fec57515c090b328e1c88d85237be6e8412cd41ff83017e917b1d00f5412c9990b8183edfd3ceaaefc4b58599935d6163ad604b023e56f100d505ccca012db8c7149b306910&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:41:08.68396866+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062942Z&X-Goog-Expires=86399&X-Goog-Signature=2f0d6f5f9c5c89e54bc0d24076ee3ada852cc3d8927e4f1458c329da822f5b1ef992e61fb14afbb502784e633001aee460c1e6b41f9c08d0e95ed7c3672da7dfb8599fa9e963604890d5823db21bcac112368aede49bb340e45f7a2ae05b73c11e58fca0585abbae4ca6c6fdbb6bd5124551ff9eb94ce975e2970f4c330af812c20df1aadddbe0cdd69b98ff8c6f0ac38b5c63aea339dfe9e0d708634f6c82a7a9ec197f7ed2fa1f4d086a7249b559e2bee1d7bfa5f06c9539b6d015668b1cd65a2ae802066366a3d43e66a4ebf891ba39e014824efb217751f0b3ef33eca0abb2fd38b72e897d2d73a1f1037f5bbbe0cce63731dad67f458704310ed90883c7&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:60998->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062942Z&X-Goog-Expires=86399&X-Goog-Signature=2f0d6f5f9c5c89e54bc0d24076ee3ada852cc3d8927e4f1458c329da822f5b1ef992e61fb14afbb502784e633001aee460c1e6b41f9c08d0e95ed7c3672da7dfb8599fa9e963604890d5823db21bcac112368aede49bb340e45f7a2ae05b73c11e58fca0585abbae4ca6c6fdbb6bd5124551ff9eb94ce975e2970f4c330af812c20df1aadddbe0cdd69b98ff8c6f0ac38b5c63aea339dfe9e0d708634f6c82a7a9ec197f7ed2fa1f4d086a7249b559e2bee1d7bfa5f06c9539b6d015668b1cd65a2ae802066366a3d43e66a4ebf891ba39e014824efb217751f0b3ef33eca0abb2fd38b72e897d2d73a1f1037f5bbbe0cce63731dad67f458704310ed90883c7&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:41:16.866321528+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062946Z&X-Goog-Expires=86399&X-Goog-Signature=a9fb26867862d7a872be0fe3943b1c45fa69319d509095ad8711be6c25366fe74232e8c8d4206f570068fde288c489ba063fbcd62bc5b95346a8b7d4944d3bc9b7851a92d2cb31647084284af54cb078ea6ead75199b810ef4ed2c49532395be0fb8446e76e25d84440cad7825f70805891e6caf9ffab8dff220311a0d61928d3568c8be34b5223bb49f33d70f8cd9cb98e38c7198f94666243187d53e91a6d9f9c587b073362486fc9d2e0c32a7b254b874b26dd04e48fb1d5f3363547f2fbd353f23ba7430f8862bed5c51094227543427c5903d7b91f8b82fb1430be2359ab81a144263d199050eb36464deb35b28fd386bb04bd9350e7f4434ca7d1b4691&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:32776->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/iunrz6lf/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T062946Z&X-Goog-Expires=86399&X-Goog-Signature=a9fb26867862d7a872be0fe3943b1c45fa69319d509095ad8711be6c25366fe74232e8c8d4206f570068fde288c489ba063fbcd62bc5b95346a8b7d4944d3bc9b7851a92d2cb31647084284af54cb078ea6ead75199b810ef4ed2c49532395be0fb8446e76e25d84440cad7825f70805891e6caf9ffab8dff220311a0d61928d3568c8be34b5223bb49f33d70f8cd9cb98e38c7198f94666243187d53e91a6d9f9c587b073362486fc9d2e0c32a7b254b874b26dd04e48fb1d5f3363547f2fbd353f23ba7430f8862bed5c51094227543427c5903d7b91f8b82fb1430be2359ab81a144263d199050eb36464deb35b28fd386bb04bd9350e7f4434ca7d1b4691&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} diff --git a/BioReason_new/wandb/run-20250813_142935-iunrz6lf/logs/debug.log b/BioReason_new/wandb/run-20250813_142935-iunrz6lf/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..635c125833e2b42799ee57d91dcf453094ac4d78 --- /dev/null +++ b/BioReason_new/wandb/run-20250813_142935-iunrz6lf/logs/debug.log @@ -0,0 +1,6780 @@ +2025-08-13 14:29:35,137 INFO MainThread:8595 [wandb_setup.py:_flush():80] Current SDK version is 0.21.1 +2025-08-13 14:29:35,137 INFO MainThread:8595 [wandb_setup.py:_flush():80] Configure stats pid to 8595 +2025-08-13 14:29:35,137 INFO MainThread:8595 [wandb_setup.py:_flush():80] Loading settings from /root/.config/wandb/settings +2025-08-13 14:29:35,137 INFO MainThread:8595 [wandb_setup.py:_flush():80] Loading settings from /nas/shared/kilab/wangyujia/BioReason_new/wandb/settings +2025-08-13 14:29:35,137 INFO MainThread:8595 [wandb_setup.py:_flush():80] Loading settings from environment variables +2025-08-13 14:29:35,137 INFO MainThread:8595 [wandb_init.py:setup_run_log_directory():703] Logging user logs to /nas/shared/kilab/wangyujia/BioReason_new/wandb/run-20250813_142935-iunrz6lf/logs/debug.log +2025-08-13 14:29:35,137 INFO MainThread:8595 [wandb_init.py:setup_run_log_directory():704] Logging internal logs to /nas/shared/kilab/wangyujia/BioReason_new/wandb/run-20250813_142935-iunrz6lf/logs/debug-internal.log +2025-08-13 14:29:35,137 INFO MainThread:8595 [wandb_init.py:init():830] calling init triggers +2025-08-13 14:29:35,138 INFO MainThread:8595 [wandb_init.py:init():835] wandb.init called with sweep_config: {} +config: {'text_model_name': '/oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged', 'protein_model_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m', 'qformer_model_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft', 'cache_dir': '/model-weights', 'num_query_tokens': 8, 'train_dataset': '/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl', 'valid_dataset': '/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl', 'eval_dataset': True, 'output_dir': './contrastive_outputs', 'num_epochs': 10, 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 0.01, 'warmup_steps': 1000, 'gradient_accumulation_steps': 1, 'temperature': 0.07, 'freeze_protein_model': True, 'freeze_text_model': True, 'protein_weight': 1.0, 'text_weight': 1.0, 'enable_ptm': True, 'ptm_weight': 1.0, 'max_length_protein': 1024, 'max_length_text': 512, 'num_workers': 8, 'logging_steps': 100, 'eval_steps': 500, 'save_steps': 1000, 'save_total_limit': 3, 'fp16': False, 'bf16': False, 'seed': 42, 'use_wandb': True, 'wandb_project': 'protein-llm-contrastive', 'wandb_entity': None, '_wandb': {}} +2025-08-13 14:29:35,138 INFO MainThread:8595 [wandb_init.py:init():871] starting backend +2025-08-13 14:29:35,345 INFO MainThread:8595 [wandb_init.py:init():874] sending inform_init request +2025-08-13 14:29:35,351 INFO MainThread:8595 [wandb_init.py:init():882] backend started and connected +2025-08-13 14:29:35,352 INFO MainThread:8595 [wandb_init.py:init():953] updated telemetry +2025-08-13 14:29:35,382 INFO MainThread:8595 [wandb_init.py:init():977] communicating run to backend with 90.0 second timeout +2025-08-13 14:29:42,201 INFO MainThread:8595 [wandb_init.py:init():1029] starting run threads in backend +2025-08-13 14:29:42,309 INFO MainThread:8595 [wandb_run.py:_console_start():2494] atexit reg +2025-08-13 14:29:42,309 INFO MainThread:8595 [wandb_run.py:_redirect():2342] redirect: wrap_raw +2025-08-13 14:29:42,310 INFO MainThread:8595 [wandb_run.py:_redirect():2411] Wrapping output streams. +2025-08-13 14:29:42,310 INFO MainThread:8595 [wandb_run.py:_redirect():2434] Redirects installed. +2025-08-13 14:29:42,312 INFO MainThread:8595 [wandb_init.py:init():1075] run started, returning control to user process +2025-08-13 14:29:45,374 INFO MsgRouterThr:8595 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles. +2025-08-13 14:44:14,681 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,685 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,686 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,687 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,687 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,688 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,688 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,688 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,689 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,689 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,690 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,690 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,691 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,691 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,692 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,692 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,692 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,693 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,693 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,694 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,694 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,694 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,695 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,695 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,695 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,696 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,696 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,697 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,697 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,697 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,698 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,698 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,698 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,699 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,699 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,699 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,700 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,700 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,701 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,701 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,701 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,702 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,702 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,703 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,703 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,703 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,704 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,704 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,705 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,705 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,705 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,706 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,706 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,706 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,707 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,707 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,708 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,708 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,708 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,709 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,709 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,709 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,710 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,710 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,710 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,711 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,711 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,712 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,712 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,712 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,713 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,713 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,714 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,714 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,714 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,715 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,715 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,716 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,716 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,717 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,717 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,717 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,718 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,718 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,719 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,719 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,719 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,720 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,720 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,720 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,721 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,721 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,721 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,722 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,722 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,723 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,723 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,723 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,724 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,724 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,724 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,725 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,725 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,725 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,726 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,726 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,727 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,727 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,727 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,728 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,728 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,728 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,729 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,729 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,729 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,730 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,730 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,730 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,731 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,731 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,732 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,732 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,733 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,733 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,733 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,734 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,734 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,735 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,735 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,735 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,736 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,736 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,736 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,737 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,737 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,738 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,738 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,738 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,739 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,739 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,739 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,740 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,740 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,740 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,741 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,741 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,741 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,742 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,742 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,743 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,743 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,743 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,744 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,744 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,744 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,745 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,745 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,745 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,746 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,746 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,746 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,747 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,747 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,748 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,748 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,749 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,749 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,749 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,750 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,750 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,750 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,751 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,751 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,752 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,752 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,752 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,753 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,753 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,753 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,754 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,754 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,755 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,755 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,755 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,756 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,756 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,756 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,757 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,757 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,757 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,758 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,758 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,758 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,759 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,759 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,760 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,760 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,760 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,761 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,761 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,761 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,762 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,762 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,762 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,763 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,763 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,763 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,764 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,764 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,764 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,765 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,765 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,766 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,766 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,766 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,767 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,767 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:44:14,767 ERROR MainThread:8595 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe diff --git a/BioReason_new/wandb/run-20250813_142935-iunrz6lf/run-iunrz6lf.wandb b/BioReason_new/wandb/run-20250813_142935-iunrz6lf/run-iunrz6lf.wandb new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/BioReason_new/wandb/run-20250813_144434-b7u1a6df/files/config.yaml b/BioReason_new/wandb/run-20250813_144434-b7u1a6df/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f66dd8f1459acef139bab7c9be6d245079bd14fc --- /dev/null +++ b/BioReason_new/wandb/run-20250813_144434-b7u1a6df/files/config.yaml @@ -0,0 +1,200 @@ +_wandb: + value: + cli_version: 0.21.1 + e: + edi3r783ioc6ivvvz46x2uehlkkl4uku: + args: + - --text_model_name + - /oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged + - --protein_model_name + - /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m + - --qformer_model_name + - /nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft + - --num_query_tokens + - "8" + - --train_dataset + - /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl + - --valid_dataset + - /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl + - --output_dir + - ./contrastive_outputs + - --num_epochs + - "10" + - --batch_size + - "8" + - --learning_rate + - "1e-4" + - --temperature + - "0.07" + - --freeze_protein_model + - --freeze_text_model + - --enable_ptm + - --max_length_protein + - "1024" + - --max_length_text + - "512" + - --num_workers + - "8" + - --eval_dataset + - --use_wandb + - --wandb_project + - protein-llm-contrastive + - --logging_steps + - "100" + - --eval_steps + - "500" + - --save_steps + - "1000" + codePath: wangyujia/BioReason_new/train_contrastive.py + codePathLocal: train_contrastive.py + cpu_count: 64 + cpu_count_logical: 64 + cudaVersion: "12.1" + disk: + /: + total: "1623302262784" + used: "1015132160" + email: gia0603yucca@gmail.com + executable: /root/miniconda3/envs/bioreason/bin/python + git: + commit: b8caf406aa1699c788f0ca6e44a1769452c317db + remote: https://github.com/PorUna-byte/PAR.git + gpu: NVIDIA A800-SXM4-80GB + gpu_count: 8 + gpu_nvidia: + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-62fcc532-880f-7324-eb13-3b739a38c29b + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-af1197aa-3e7d-a53e-185d-04b1d64164da + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-3e6e551f-2d7d-a27b-654a-da56933d2283 + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-a947b64a-c545-ee5e-30e6-b4b0e2dd8fdc + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-ebd04f3f-c437-1b8b-26f2-4b1ebfee93c4 + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-53994891-72a6-25be-a4c8-e32c695155b0 + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-a927d00a-15c5-31f5-9ae1-9d2e773821a1 + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-93d555c4-29a0-cd78-abac-d98cd160d4e1 + host: dsw-265304-585cc9d768-ckfd7 + memory: + total: "549755813888" + os: Linux-5.10.134-008.18.kangaroo.al8.x86_64-x86_64-with-glibc2.35 + program: /nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py + python: CPython 3.11.0 + root: /nas/shared/kilab/wangyujia/BioReason_new + startedAt: "2025-08-13T06:44:34.902815Z" + writerId: edi3r783ioc6ivvvz46x2uehlkkl4uku + m: [] + python_version: 3.11.0 + t: + "1": + - 1 + - 9 + - 11 + - 41 + - 49 + - 51 + - 71 + - 84 + - 98 + - 103 + "2": + - 1 + - 9 + - 11 + - 41 + - 49 + - 51 + - 71 + - 84 + - 98 + - 103 + "3": + - 13 + - 16 + "4": 3.11.0 + "5": 0.21.1 + "6": 4.55.0 + "12": 0.21.1 + "13": linux-x86_64 +batch_size: + value: 8 +bf16: + value: false +cache_dir: + value: /model-weights +enable_ptm: + value: true +eval_dataset: + value: true +eval_steps: + value: 500 +fp16: + value: false +freeze_protein_model: + value: true +freeze_text_model: + value: true +gradient_accumulation_steps: + value: 1 +learning_rate: + value: 0.0001 +logging_steps: + value: 100 +max_length_protein: + value: 1024 +max_length_text: + value: 512 +num_epochs: + value: 10 +num_query_tokens: + value: 8 +num_workers: + value: 8 +output_dir: + value: ./contrastive_outputs +protein_model_name: + value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m +protein_weight: + value: 1 +ptm_weight: + value: 1 +qformer_model_name: + value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft +save_steps: + value: 1000 +save_total_limit: + value: 3 +seed: + value: 42 +temperature: + value: 0.07 +text_model_name: + value: /oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged +text_weight: + value: 1 +train_dataset: + value: /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl +use_wandb: + value: true +valid_dataset: + value: /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl +wandb_entity: + value: null +wandb_project: + value: protein-llm-contrastive +warmup_steps: + value: 1000 +weight_decay: + value: 0.01 diff --git a/BioReason_new/wandb/run-20250813_144434-b7u1a6df/files/output.log b/BioReason_new/wandb/run-20250813_144434-b7u1a6df/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..dc53e4c6aef218f59fb67e8dd63785cf706a8e3c --- /dev/null +++ b/BioReason_new/wandb/run-20250813_144434-b7u1a6df/files/output.log @@ -0,0 +1,40 @@ +Loading model... +Loading checkpoint shards: 100%|██████████████████████████████████████████████████████| 4/4 [00:01<00:00, 2.62it/s] +Some weights of EsmModel were not initialized from the model checkpoint at /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m and are newly initialized: ['pooler.dense.bias', 'pooler.dense.weight'] +You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference. +Traceback (most recent call last): + File "/nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py", line 552, in + trainer = main(args) + ^^^^^^^^^^ + File "/nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py", line 303, in main + model = ProteinLLMModel( + ^^^^^^^^^^^^^^^^ + File "/nas/shared/kilab/wangyujia/BioReason_new/bioreason/models/protein_llm.py", line 216, in __init__ + self.protein_projection = QFormerProjector( + ^^^^^^^^^^^^^^^^^ + File "/nas/shared/kilab/wangyujia/BioReason_new/bioreason/models/protein_llm.py", line 63, in __init__ + self.qformer = BertModel.from_pretrained(qformer_model_name, config=config) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/modeling_utils.py", line 316, in _wrapper + return func(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/modeling_utils.py", line 5061, in from_pretrained + ) = cls._load_pretrained_model( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/modeling_utils.py", line 5524, in _load_pretrained_model + _error_msgs, disk_offload_index, cpu_offload_index = load_shard_file(args) + ^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/modeling_utils.py", line 974, in load_shard_file + disk_offload_index, cpu_offload_index = _load_state_dict_into_meta_model( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context + return func(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/modeling_utils.py", line 879, in _load_state_dict_into_meta_model + _load_parameter_into_model(model, param_name, param.to(param_device)) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/modeling_utils.py", line 739, in _load_parameter_into_model + module.load_state_dict({param_type: tensor}, strict=False, assign=True) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/torch/nn/modules/module.py", line 2593, in load_state_dict + raise RuntimeError( +RuntimeError: Error(s) in loading state_dict for Embedding: + size mismatch for weight: copying a param with shape torch.Size([512, 768]) from checkpoint, the shape in current model is torch.Size([1024, 768]). diff --git a/BioReason_new/wandb/run-20250813_144434-b7u1a6df/files/requirements.txt b/BioReason_new/wandb/run-20250813_144434-b7u1a6df/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..743d03fe27e0b7e200935023c07a1d7fa61b99ce --- /dev/null +++ b/BioReason_new/wandb/run-20250813_144434-b7u1a6df/files/requirements.txt @@ -0,0 +1,233 @@ +jupyterlab_pygments==0.3.0 +depyf==0.19.0 +av==15.0.0 +fastapi==0.116.1 +ray==2.48.0 +cachetools==6.1.0 +prometheus_client==0.22.1 +requests==2.32.4 +jupyter-lsp==2.2.6 +jupyter-events==0.12.0 +anyio==4.10.0 +pybase64==1.4.2 +fastjsonschema==2.21.1 +cloudpickle==3.1.1 +sentry-sdk==2.34.1 +torchmetrics==1.8.1 +wheel==0.45.1 +idna==3.10 +defusedxml==0.7.1 +starlette==0.47.2 +python-dotenv==1.1.1 +comm==0.2.3 +multidict==6.6.3 +python-json-logger==3.3.0 +jupyter_server==2.16.0 +nvidia-nvtx-cu12==12.6.77 +websockets==15.0.1 +referencing==0.36.2 +gguf==0.17.1 +lm-format-enforcer==0.10.12 +xformers==0.0.31 +ninja==1.11.1.4 +beautifulsoup4==4.13.4 +dnspython==2.7.0 +certifi==2025.8.3 +markdown-it-py==3.0.0 +stack-data==0.6.3 +nvidia-cusparselt-cu12==0.6.3 +networkx==3.5 +aiohappyeyeballs==2.6.1 +nvidia-cuda-cupti-cu12==12.6.80 +llguidance==0.7.30 +rich==14.1.0 +outlines_core==0.2.10 +nvidia-nccl-cu12==2.26.2 +pandas==2.3.1 +pydantic_core==2.33.2 +argon2-cffi==25.1.0 +babel==2.17.0 +isoduration==20.11.0 +uvloop==0.21.0 +json5==0.12.0 +widgetsnbextension==4.0.14 +aiohttp==3.12.15 +notebook_shim==0.2.4 +pandocfilters==1.5.1 +vllm==0.10.0 +blake3==1.0.5 +diskcache==5.6.3 +h11==0.16.0 +attrs==25.3.0 +nvidia-cusolver-cu12==11.7.1.2 +click==8.2.1 +cupy-cuda12x==13.5.1 +jedi==0.19.2 +py-cpuinfo==9.0.0 +overrides==7.7.0 +pytz==2025.2 +protobuf==6.31.1 +dill==0.3.8 +ipykernel==6.30.1 +frozenlist==1.7.0 +huggingface-hub==0.34.4 +rignore==0.6.4 +mistune==3.1.3 +sniffio==1.3.1 +httpx==0.28.1 +sentencepiece==0.2.0 +typing_extensions==4.14.1 +typing-inspection==0.4.1 +async-lru==2.0.5 +bioreason==0.1.0 +charset-normalizer==3.4.3 +llvmlite==0.44.0 +safetensors==0.6.2 +fqdn==1.5.1 +peft==0.17.0 +fastapi-cli==0.0.8 +torch==2.7.1 +jsonschema==4.25.0 +torchvision==0.22.1 +parso==0.8.4 +pillow==11.3.0 +pycountry==24.6.1 +scipy==1.16.1 +wandb==0.21.1 +pip==25.1 +wcwidth==0.2.13 +mistral_common==1.8.3 +nvidia-cudnn-cu12==9.5.1.17 +asttokens==3.0.0 +numpy==2.2.6 +uvicorn==0.35.0 +jupyter_client==8.6.3 +debugpy==1.8.16 +fastapi-cloud-cli==0.1.5 +compressed-tensors==0.10.2 +GitPython==3.1.45 +nvidia-ml-py==13.580.65 +tiktoken==0.11.0 +partial-json-parser==0.2.1.1.post6 +prompt_toolkit==3.0.51 +uri-template==1.3.0 +pyarrow==21.0.0 +nvidia-cuda-nvrtc-cu12==12.6.77 +propcache==0.3.2 +matplotlib-inline==0.1.7 +pure_eval==0.2.3 +nest-asyncio==1.6.0 +deepspeed==0.17.4 +einops==0.8.1 +triton==3.3.1 +trl==0.21.0 +pyzmq==27.0.1 +httptools==0.6.4 +ipython==9.4.0 +packaging==25.0 +hjson==3.1.0 +soxr==0.5.0.post1 +webcolors==24.11.1 +PyYAML==6.0.2 +ipywidgets==8.1.7 +lark==1.2.2 +python-multipart==0.0.20 +tinycss2==1.4.0 +mpmath==1.3.0 +psutil==7.0.0 +nbformat==5.10.4 +qwen-vl-utils==0.0.11 +notebook==7.4.5 +jupyterlab==4.4.5 +transformers==4.55.0 +urllib3==2.5.0 +terminado==0.18.1 +xgrammar==0.1.21 +xxhash==3.5.0 +tzdata==2025.2 +jupyterlab_widgets==3.0.15 +aiosignal==1.4.0 +annotated-types==0.7.0 +nvidia-nvjitlink-cu12==12.6.85 +httpcore==1.0.9 +pexpect==4.9.0 +tornado==6.5.2 +regex==2025.7.33 +filelock==3.18.0 +nvidia-cuda-runtime-cu12==12.6.77 +openai==1.90.0 +Pygments==2.19.2 +cffi==1.17.1 +nvidia-cufft-cu12==11.3.0.4 +rpds-py==0.27.0 +rich-toolkit==0.15.0 +bleach==6.2.0 +sympy==1.14.0 +python-dateutil==2.9.0.post0 +watchfiles==1.1.0 +nbclient==0.10.2 +hf-xet==1.1.7 +Send2Trash==1.8.3 +smmap==5.0.2 +tokenizers==0.21.4 +yarl==1.20.1 +MarkupSafe==3.0.2 +prometheus-fastapi-instrumentator==7.1.0 +rfc3986-validator==0.1.1 +bitsandbytes==0.46.1 +nvidia-cufile-cu12==1.11.1.6 +fsspec==2025.3.0 +executing==2.2.0 +pydantic==2.11.7 +jupyter_core==5.8.1 +msgspec==0.19.0 +nvidia-cusparse-cu12==12.5.4.2 +opencv-python-headless==4.12.0.88 +webencodings==0.5.1 +ptyprocess==0.7.0 +pytorch-lightning==2.5.2 +Jinja2==3.1.6 +websocket-client==1.8.0 +jsonpointer==3.0.0 +traitlets==5.14.3 +numba==0.61.2 +soupsieve==2.7 +cbor2==5.6.5 +multiprocess==0.70.16 +email_validator==2.2.0 +distro==1.9.0 +accelerate==1.10.0 +torchaudio==2.7.1 +jiter==0.10.0 +rfc3339-validator==0.1.4 +jupyter==1.1.1 +typer==0.16.0 +gitdb==4.0.12 +types-python-dateutil==2.9.0.20250809 +soundfile==0.13.1 +astor==0.8.1 +shellingham==1.5.4 +jupyterlab_server==2.27.3 +nvidia-curand-cu12==10.3.7.77 +jupyter-console==6.6.3 +msgpack==1.1.1 +six==1.17.0 +pycparser==2.22 +arrow==1.3.0 +tqdm==4.67.1 +lightning-utilities==0.15.2 +interegular==0.3.3 +rfc3987-syntax==1.1.0 +ipython_pygments_lexers==1.1.1 +datasets==4.0.0 +jupyter_server_terminals==0.5.3 +platformdirs==4.3.8 +pydantic-extra-types==2.10.5 +decorator==5.2.1 +mdurl==0.1.2 +nvidia-cublas-cu12==12.6.4.1 +jsonschema-specifications==2025.4.1 +setuptools==78.1.1 +fastrlock==0.8.3 +argon2-cffi-bindings==25.1.0 +nbconvert==7.16.6 diff --git a/BioReason_new/wandb/run-20250813_144434-b7u1a6df/files/wandb-metadata.json b/BioReason_new/wandb/run-20250813_144434-b7u1a6df/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..89fe8bfa6b9274ae0c2596106db3ff6189d43b07 --- /dev/null +++ b/BioReason_new/wandb/run-20250813_144434-b7u1a6df/files/wandb-metadata.json @@ -0,0 +1,116 @@ +{ + "os": "Linux-5.10.134-008.18.kangaroo.al8.x86_64-x86_64-with-glibc2.35", + "python": "CPython 3.11.0", + "startedAt": "2025-08-13T06:44:34.902815Z", + "args": [ + "--text_model_name", + "/oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged", + "--protein_model_name", + "/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m", + "--qformer_model_name", + "/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft", + "--num_query_tokens", + "8", + "--train_dataset", + "/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl", + "--valid_dataset", + "/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl", + "--output_dir", + "./contrastive_outputs", + "--num_epochs", + "10", + "--batch_size", + "8", + "--learning_rate", + "1e-4", + "--temperature", + "0.07", + "--freeze_protein_model", + "--freeze_text_model", + "--enable_ptm", + "--max_length_protein", + "1024", + "--max_length_text", + "512", + "--num_workers", + "8", + "--eval_dataset", + "--use_wandb", + "--wandb_project", + "protein-llm-contrastive", + "--logging_steps", + "100", + "--eval_steps", + "500", + "--save_steps", + "1000" + ], + "program": "/nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py", + "codePath": "wangyujia/BioReason_new/train_contrastive.py", + "codePathLocal": "train_contrastive.py", + "git": { + "remote": "https://github.com/PorUna-byte/PAR.git", + "commit": "b8caf406aa1699c788f0ca6e44a1769452c317db" + }, + "email": "gia0603yucca@gmail.com", + "root": "/nas/shared/kilab/wangyujia/BioReason_new", + "host": "dsw-265304-585cc9d768-ckfd7", + "executable": "/root/miniconda3/envs/bioreason/bin/python", + "cpu_count": 64, + "cpu_count_logical": 64, + "gpu": "NVIDIA A800-SXM4-80GB", + "gpu_count": 8, + "disk": { + "/": { + "total": "1623302262784", + "used": "1015132160" + } + }, + "memory": { + "total": "549755813888" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-62fcc532-880f-7324-eb13-3b739a38c29b" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-af1197aa-3e7d-a53e-185d-04b1d64164da" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-3e6e551f-2d7d-a27b-654a-da56933d2283" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-a947b64a-c545-ee5e-30e6-b4b0e2dd8fdc" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-ebd04f3f-c437-1b8b-26f2-4b1ebfee93c4" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-53994891-72a6-25be-a4c8-e32c695155b0" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-a927d00a-15c5-31f5-9ae1-9d2e773821a1" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-93d555c4-29a0-cd78-abac-d98cd160d4e1" + } + ], + "cudaVersion": "12.1", + "writerId": "edi3r783ioc6ivvvz46x2uehlkkl4uku" +} \ No newline at end of file diff --git a/BioReason_new/wandb/run-20250813_144434-b7u1a6df/files/wandb-summary.json b/BioReason_new/wandb/run-20250813_144434-b7u1a6df/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..75631e6e43bf8471ee6ca35a1ab1286f569677cb --- /dev/null +++ b/BioReason_new/wandb/run-20250813_144434-b7u1a6df/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":3},"_runtime":3} \ No newline at end of file diff --git a/BioReason_new/wandb/run-20250813_144434-b7u1a6df/logs/debug-internal.log b/BioReason_new/wandb/run-20250813_144434-b7u1a6df/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..adc0daba817fca97eaa81e6a577569255f97199d --- /dev/null +++ b/BioReason_new/wandb/run-20250813_144434-b7u1a6df/logs/debug-internal.log @@ -0,0 +1,33 @@ +{"time":"2025-08-13T14:44:35.175839156+08:00","level":"INFO","msg":"stream: starting","core version":"0.21.1"} +{"time":"2025-08-13T14:44:37.280573541+08:00","level":"INFO","msg":"stream: created new stream","id":"b7u1a6df"} +{"time":"2025-08-13T14:44:37.280624869+08:00","level":"INFO","msg":"stream: started","id":"b7u1a6df"} +{"time":"2025-08-13T14:44:37.28063665+08:00","level":"INFO","msg":"handler: started","stream_id":"b7u1a6df"} +{"time":"2025-08-13T14:44:37.280650129+08:00","level":"INFO","msg":"sender: started","stream_id":"b7u1a6df"} +{"time":"2025-08-13T14:44:37.281034381+08:00","level":"INFO","msg":"writer: started","stream_id":"b7u1a6df"} +{"time":"2025-08-13T14:44:39.434424587+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064438Z&X-Goog-Expires=86399&X-Goog-Signature=1c4d8e69c4a4bc16a1ecff0d2fc584e4c3edf928e99bc626ec3d49efbad4153386ec6f02ecfb9a713d077b0e7533a5e727b8841e1f183aaa50bc27bdb38788135973a93bd004290294d6805b7dead160711dcde78af1548be3f499ef000c710b6088b024dc1f374aa65daae277aa91ff8acb08d522905678e0f093d5e15a6a5d353ff3b07e46494349753f29051a19a8b88b453cf21b66006a30a00a1c086288b991685b1aa294e714697f4eee27b2175e9fcfb3c0e19cf2c191a0414c7d7c4d8d3e27a54fa2b509a430977dfc12087aa7109df8e6f999dc0c21b1f79d1973e5f6e496b36fecdef5039685395b130a02c153b53cde1288da4a2618f61b8f6f03&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:46502->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064438Z&X-Goog-Expires=86399&X-Goog-Signature=1c4d8e69c4a4bc16a1ecff0d2fc584e4c3edf928e99bc626ec3d49efbad4153386ec6f02ecfb9a713d077b0e7533a5e727b8841e1f183aaa50bc27bdb38788135973a93bd004290294d6805b7dead160711dcde78af1548be3f499ef000c710b6088b024dc1f374aa65daae277aa91ff8acb08d522905678e0f093d5e15a6a5d353ff3b07e46494349753f29051a19a8b88b453cf21b66006a30a00a1c086288b991685b1aa294e714697f4eee27b2175e9fcfb3c0e19cf2c191a0414c7d7c4d8d3e27a54fa2b509a430977dfc12087aa7109df8e6f999dc0c21b1f79d1973e5f6e496b36fecdef5039685395b130a02c153b53cde1288da4a2618f61b8f6f03&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:44:40.314088388+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064438Z&X-Goog-Expires=86399&X-Goog-Signature=b784e18b1e51706c965d5e39b31f26a5ab282191d4e34eba448ded3882706352d7cd0eb8afc78f4f6585ac2292839aa84719202f49c853ee4b4d5930137d15a5ca9c3bf336b9196fa43ea17730c012e474eb78da8dd53632305b7b46e7ce9224d376e786f4d9c6361a4c65201c804642343ee4933f6a37d8f230d25e497858b2a762c750d9b71911cc81cd8d447af1d7d384833af2cf8211ac7d6dc4b05f8b4219b6779f3e89d047d1072c8c583e36cafbb7d8151f915d1f214952244834adb7cf57459c7f14af22d01437ce43c677279de3bb3f7d40ee64b49823294ba1f30530d885cda6864edfc4e27cd2ccb101e3ca1e44f43de80cb7377595deea1aec72&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:46512->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064438Z&X-Goog-Expires=86399&X-Goog-Signature=b784e18b1e51706c965d5e39b31f26a5ab282191d4e34eba448ded3882706352d7cd0eb8afc78f4f6585ac2292839aa84719202f49c853ee4b4d5930137d15a5ca9c3bf336b9196fa43ea17730c012e474eb78da8dd53632305b7b46e7ce9224d376e786f4d9c6361a4c65201c804642343ee4933f6a37d8f230d25e497858b2a762c750d9b71911cc81cd8d447af1d7d384833af2cf8211ac7d6dc4b05f8b4219b6779f3e89d047d1072c8c583e36cafbb7d8151f915d1f214952244834adb7cf57459c7f14af22d01437ce43c677279de3bb3f7d40ee64b49823294ba1f30530d885cda6864edfc4e27cd2ccb101e3ca1e44f43de80cb7377595deea1aec72&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:44:41.741016978+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064438Z&X-Goog-Expires=86399&X-Goog-Signature=1c4d8e69c4a4bc16a1ecff0d2fc584e4c3edf928e99bc626ec3d49efbad4153386ec6f02ecfb9a713d077b0e7533a5e727b8841e1f183aaa50bc27bdb38788135973a93bd004290294d6805b7dead160711dcde78af1548be3f499ef000c710b6088b024dc1f374aa65daae277aa91ff8acb08d522905678e0f093d5e15a6a5d353ff3b07e46494349753f29051a19a8b88b453cf21b66006a30a00a1c086288b991685b1aa294e714697f4eee27b2175e9fcfb3c0e19cf2c191a0414c7d7c4d8d3e27a54fa2b509a430977dfc12087aa7109df8e6f999dc0c21b1f79d1973e5f6e496b36fecdef5039685395b130a02c153b53cde1288da4a2618f61b8f6f03&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:45294->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064438Z&X-Goog-Expires=86399&X-Goog-Signature=1c4d8e69c4a4bc16a1ecff0d2fc584e4c3edf928e99bc626ec3d49efbad4153386ec6f02ecfb9a713d077b0e7533a5e727b8841e1f183aaa50bc27bdb38788135973a93bd004290294d6805b7dead160711dcde78af1548be3f499ef000c710b6088b024dc1f374aa65daae277aa91ff8acb08d522905678e0f093d5e15a6a5d353ff3b07e46494349753f29051a19a8b88b453cf21b66006a30a00a1c086288b991685b1aa294e714697f4eee27b2175e9fcfb3c0e19cf2c191a0414c7d7c4d8d3e27a54fa2b509a430977dfc12087aa7109df8e6f999dc0c21b1f79d1973e5f6e496b36fecdef5039685395b130a02c153b53cde1288da4a2618f61b8f6f03&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:44:41.989817652+08:00","level":"INFO","msg":"stream: closing","id":"b7u1a6df"} +{"time":"2025-08-13T14:44:42.777303637+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064438Z&X-Goog-Expires=86399&X-Goog-Signature=b784e18b1e51706c965d5e39b31f26a5ab282191d4e34eba448ded3882706352d7cd0eb8afc78f4f6585ac2292839aa84719202f49c853ee4b4d5930137d15a5ca9c3bf336b9196fa43ea17730c012e474eb78da8dd53632305b7b46e7ce9224d376e786f4d9c6361a4c65201c804642343ee4933f6a37d8f230d25e497858b2a762c750d9b71911cc81cd8d447af1d7d384833af2cf8211ac7d6dc4b05f8b4219b6779f3e89d047d1072c8c583e36cafbb7d8151f915d1f214952244834adb7cf57459c7f14af22d01437ce43c677279de3bb3f7d40ee64b49823294ba1f30530d885cda6864edfc4e27cd2ccb101e3ca1e44f43de80cb7377595deea1aec72&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:51458->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064438Z&X-Goog-Expires=86399&X-Goog-Signature=b784e18b1e51706c965d5e39b31f26a5ab282191d4e34eba448ded3882706352d7cd0eb8afc78f4f6585ac2292839aa84719202f49c853ee4b4d5930137d15a5ca9c3bf336b9196fa43ea17730c012e474eb78da8dd53632305b7b46e7ce9224d376e786f4d9c6361a4c65201c804642343ee4933f6a37d8f230d25e497858b2a762c750d9b71911cc81cd8d447af1d7d384833af2cf8211ac7d6dc4b05f8b4219b6779f3e89d047d1072c8c583e36cafbb7d8151f915d1f214952244834adb7cf57459c7f14af22d01437ce43c677279de3bb3f7d40ee64b49823294ba1f30530d885cda6864edfc4e27cd2ccb101e3ca1e44f43de80cb7377595deea1aec72&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:44:46.722453063+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064443Z&X-Goog-Expires=86399&X-Goog-Signature=a1ed9a3895a7355448222045946c9063c3c6f485bd42fdbf0f75878011690b13c231c0d4aca1dc09a5e1858884868f833ed37fe1c60079b13bd9b6bfc8819178c67a00fab449892d0eb5d54823719610deefa2051881eeed0f85c20a794ae5fbc58c588cb260634bf12a6e9adabcb55dcaa39014e44df8f6e83cdb24b4e514a211d9981878a93b681e87792d5e3d8c92786a6ba10ab50a7731f91112fbdddef9aad498d4c0513b08d85241958614cbd19074b46becfe64edc60792f69a44d99612911006d970e3514eb0813dd8af1b603c91053990e61f1ef863736debc5dac74879bbed807d2e917ea1fef6257a1be0c86b9abda3db595ba8f0776d7f59639b&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:51466->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064443Z&X-Goog-Expires=86399&X-Goog-Signature=a1ed9a3895a7355448222045946c9063c3c6f485bd42fdbf0f75878011690b13c231c0d4aca1dc09a5e1858884868f833ed37fe1c60079b13bd9b6bfc8819178c67a00fab449892d0eb5d54823719610deefa2051881eeed0f85c20a794ae5fbc58c588cb260634bf12a6e9adabcb55dcaa39014e44df8f6e83cdb24b4e514a211d9981878a93b681e87792d5e3d8c92786a6ba10ab50a7731f91112fbdddef9aad498d4c0513b08d85241958614cbd19074b46becfe64edc60792f69a44d99612911006d970e3514eb0813dd8af1b603c91053990e61f1ef863736debc5dac74879bbed807d2e917ea1fef6257a1be0c86b9abda3db595ba8f0776d7f59639b&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:44:49.307483393+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064443Z&X-Goog-Expires=86399&X-Goog-Signature=a1ed9a3895a7355448222045946c9063c3c6f485bd42fdbf0f75878011690b13c231c0d4aca1dc09a5e1858884868f833ed37fe1c60079b13bd9b6bfc8819178c67a00fab449892d0eb5d54823719610deefa2051881eeed0f85c20a794ae5fbc58c588cb260634bf12a6e9adabcb55dcaa39014e44df8f6e83cdb24b4e514a211d9981878a93b681e87792d5e3d8c92786a6ba10ab50a7731f91112fbdddef9aad498d4c0513b08d85241958614cbd19074b46becfe64edc60792f69a44d99612911006d970e3514eb0813dd8af1b603c91053990e61f1ef863736debc5dac74879bbed807d2e917ea1fef6257a1be0c86b9abda3db595ba8f0776d7f59639b&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:43308->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064443Z&X-Goog-Expires=86399&X-Goog-Signature=a1ed9a3895a7355448222045946c9063c3c6f485bd42fdbf0f75878011690b13c231c0d4aca1dc09a5e1858884868f833ed37fe1c60079b13bd9b6bfc8819178c67a00fab449892d0eb5d54823719610deefa2051881eeed0f85c20a794ae5fbc58c588cb260634bf12a6e9adabcb55dcaa39014e44df8f6e83cdb24b4e514a211d9981878a93b681e87792d5e3d8c92786a6ba10ab50a7731f91112fbdddef9aad498d4c0513b08d85241958614cbd19074b46becfe64edc60792f69a44d99612911006d970e3514eb0813dd8af1b603c91053990e61f1ef863736debc5dac74879bbed807d2e917ea1fef6257a1be0c86b9abda3db595ba8f0776d7f59639b&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:44:53.695162885+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064443Z&X-Goog-Expires=86399&X-Goog-Signature=a1ed9a3895a7355448222045946c9063c3c6f485bd42fdbf0f75878011690b13c231c0d4aca1dc09a5e1858884868f833ed37fe1c60079b13bd9b6bfc8819178c67a00fab449892d0eb5d54823719610deefa2051881eeed0f85c20a794ae5fbc58c588cb260634bf12a6e9adabcb55dcaa39014e44df8f6e83cdb24b4e514a211d9981878a93b681e87792d5e3d8c92786a6ba10ab50a7731f91112fbdddef9aad498d4c0513b08d85241958614cbd19074b46becfe64edc60792f69a44d99612911006d970e3514eb0813dd8af1b603c91053990e61f1ef863736debc5dac74879bbed807d2e917ea1fef6257a1be0c86b9abda3db595ba8f0776d7f59639b&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:46516->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064443Z&X-Goog-Expires=86399&X-Goog-Signature=a1ed9a3895a7355448222045946c9063c3c6f485bd42fdbf0f75878011690b13c231c0d4aca1dc09a5e1858884868f833ed37fe1c60079b13bd9b6bfc8819178c67a00fab449892d0eb5d54823719610deefa2051881eeed0f85c20a794ae5fbc58c588cb260634bf12a6e9adabcb55dcaa39014e44df8f6e83cdb24b4e514a211d9981878a93b681e87792d5e3d8c92786a6ba10ab50a7731f91112fbdddef9aad498d4c0513b08d85241958614cbd19074b46becfe64edc60792f69a44d99612911006d970e3514eb0813dd8af1b603c91053990e61f1ef863736debc5dac74879bbed807d2e917ea1fef6257a1be0c86b9abda3db595ba8f0776d7f59639b&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:46:52.722200106+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064442Z&X-Goog-Expires=86399&X-Goog-Signature=552d74aaa0c7a9ee8b16a055c4a3ae605223c81742d6231450f19f617797cdf1d95d8c626b624b1ae60476efdea6a6493422a74e44cfb29852257c4d29f9af812376ffea925f7202ef9d4e6d8b45406df628af59698be20fd994c7fc8357d6382e5db66115fd4506f8d3cc341450e297ca450ab1415621427e7e038e8298cf4693042ab314dda6c11ecacb5c31b9149ee121274a41ebf872a03f5d03bd65418bca7c8793eb46f19239ca0e2b83c5e22aeedb4ab15f310d51023c98305115916c47bc12887198169563b297c4539e2921fe7fadc5c61a3c195e7cc9420a952f614cbce034fb3e0a9c025e35797c41d4d91065e34db90fa48d47fab4198a2a9945&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:40186->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064442Z&X-Goog-Expires=86399&X-Goog-Signature=552d74aaa0c7a9ee8b16a055c4a3ae605223c81742d6231450f19f617797cdf1d95d8c626b624b1ae60476efdea6a6493422a74e44cfb29852257c4d29f9af812376ffea925f7202ef9d4e6d8b45406df628af59698be20fd994c7fc8357d6382e5db66115fd4506f8d3cc341450e297ca450ab1415621427e7e038e8298cf4693042ab314dda6c11ecacb5c31b9149ee121274a41ebf872a03f5d03bd65418bca7c8793eb46f19239ca0e2b83c5e22aeedb4ab15f310d51023c98305115916c47bc12887198169563b297c4539e2921fe7fadc5c61a3c195e7cc9420a952f614cbce034fb3e0a9c025e35797c41d4d91065e34db90fa48d47fab4198a2a9945&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:46:53.739297603+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064442Z&X-Goog-Expires=86399&X-Goog-Signature=18af461e43ed41e2c3a450ebab1acbb283be2f2273ca1ff2f8566f3de8aa061a57c9a976e1e832eb09904aa60e52c339d76234f48911b7414b9ccfb9d7e65ba092532fe77282e9c592d34435b6d111b8d91f81c1283fe85304b69972c3d5258038dcc9c06748efe8517ac504ccaca2aa5d3d83a205076eebb74f2a272e8dcc12cc93743bff75472b91f9d2be2716932a5ff40617c584c06bc4f2e9c267b0a1ad8dae6823f68c45a7f7bb22e8f84eab16df5228b51c5b6daf47084b2ebca10027f9117836dbfc8f62e2d96e159fac46bb346cbbeecaf4020adc3663d6c828828fbedebde155686bb7e2e7af86fdac8373bc807cbb1b0e53f0f2fd98e3d1a33643&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:40190->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064442Z&X-Goog-Expires=86399&X-Goog-Signature=18af461e43ed41e2c3a450ebab1acbb283be2f2273ca1ff2f8566f3de8aa061a57c9a976e1e832eb09904aa60e52c339d76234f48911b7414b9ccfb9d7e65ba092532fe77282e9c592d34435b6d111b8d91f81c1283fe85304b69972c3d5258038dcc9c06748efe8517ac504ccaca2aa5d3d83a205076eebb74f2a272e8dcc12cc93743bff75472b91f9d2be2716932a5ff40617c584c06bc4f2e9c267b0a1ad8dae6823f68c45a7f7bb22e8f84eab16df5228b51c5b6daf47084b2ebca10027f9117836dbfc8f62e2d96e159fac46bb346cbbeecaf4020adc3663d6c828828fbedebde155686bb7e2e7af86fdac8373bc807cbb1b0e53f0f2fd98e3d1a33643&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:47:13.20737949+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064443Z&X-Goog-Expires=86399&X-Goog-Signature=a1ed9a3895a7355448222045946c9063c3c6f485bd42fdbf0f75878011690b13c231c0d4aca1dc09a5e1858884868f833ed37fe1c60079b13bd9b6bfc8819178c67a00fab449892d0eb5d54823719610deefa2051881eeed0f85c20a794ae5fbc58c588cb260634bf12a6e9adabcb55dcaa39014e44df8f6e83cdb24b4e514a211d9981878a93b681e87792d5e3d8c92786a6ba10ab50a7731f91112fbdddef9aad498d4c0513b08d85241958614cbd19074b46becfe64edc60792f69a44d99612911006d970e3514eb0813dd8af1b603c91053990e61f1ef863736debc5dac74879bbed807d2e917ea1fef6257a1be0c86b9abda3db595ba8f0776d7f59639b&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:53132->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064443Z&X-Goog-Expires=86399&X-Goog-Signature=a1ed9a3895a7355448222045946c9063c3c6f485bd42fdbf0f75878011690b13c231c0d4aca1dc09a5e1858884868f833ed37fe1c60079b13bd9b6bfc8819178c67a00fab449892d0eb5d54823719610deefa2051881eeed0f85c20a794ae5fbc58c588cb260634bf12a6e9adabcb55dcaa39014e44df8f6e83cdb24b4e514a211d9981878a93b681e87792d5e3d8c92786a6ba10ab50a7731f91112fbdddef9aad498d4c0513b08d85241958614cbd19074b46becfe64edc60792f69a44d99612911006d970e3514eb0813dd8af1b603c91053990e61f1ef863736debc5dac74879bbed807d2e917ea1fef6257a1be0c86b9abda3db595ba8f0776d7f59639b&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:47:32.883795685+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064443Z&X-Goog-Expires=86399&X-Goog-Signature=a1ed9a3895a7355448222045946c9063c3c6f485bd42fdbf0f75878011690b13c231c0d4aca1dc09a5e1858884868f833ed37fe1c60079b13bd9b6bfc8819178c67a00fab449892d0eb5d54823719610deefa2051881eeed0f85c20a794ae5fbc58c588cb260634bf12a6e9adabcb55dcaa39014e44df8f6e83cdb24b4e514a211d9981878a93b681e87792d5e3d8c92786a6ba10ab50a7731f91112fbdddef9aad498d4c0513b08d85241958614cbd19074b46becfe64edc60792f69a44d99612911006d970e3514eb0813dd8af1b603c91053990e61f1ef863736debc5dac74879bbed807d2e917ea1fef6257a1be0c86b9abda3db595ba8f0776d7f59639b&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:48830->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064443Z&X-Goog-Expires=86399&X-Goog-Signature=a1ed9a3895a7355448222045946c9063c3c6f485bd42fdbf0f75878011690b13c231c0d4aca1dc09a5e1858884868f833ed37fe1c60079b13bd9b6bfc8819178c67a00fab449892d0eb5d54823719610deefa2051881eeed0f85c20a794ae5fbc58c588cb260634bf12a6e9adabcb55dcaa39014e44df8f6e83cdb24b4e514a211d9981878a93b681e87792d5e3d8c92786a6ba10ab50a7731f91112fbdddef9aad498d4c0513b08d85241958614cbd19074b46becfe64edc60792f69a44d99612911006d970e3514eb0813dd8af1b603c91053990e61f1ef863736debc5dac74879bbed807d2e917ea1fef6257a1be0c86b9abda3db595ba8f0776d7f59639b&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:49:07.902766533+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064438Z&X-Goog-Expires=86399&X-Goog-Signature=b784e18b1e51706c965d5e39b31f26a5ab282191d4e34eba448ded3882706352d7cd0eb8afc78f4f6585ac2292839aa84719202f49c853ee4b4d5930137d15a5ca9c3bf336b9196fa43ea17730c012e474eb78da8dd53632305b7b46e7ce9224d376e786f4d9c6361a4c65201c804642343ee4933f6a37d8f230d25e497858b2a762c750d9b71911cc81cd8d447af1d7d384833af2cf8211ac7d6dc4b05f8b4219b6779f3e89d047d1072c8c583e36cafbb7d8151f915d1f214952244834adb7cf57459c7f14af22d01437ce43c677279de3bb3f7d40ee64b49823294ba1f30530d885cda6864edfc4e27cd2ccb101e3ca1e44f43de80cb7377595deea1aec72&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:58408->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064438Z&X-Goog-Expires=86399&X-Goog-Signature=b784e18b1e51706c965d5e39b31f26a5ab282191d4e34eba448ded3882706352d7cd0eb8afc78f4f6585ac2292839aa84719202f49c853ee4b4d5930137d15a5ca9c3bf336b9196fa43ea17730c012e474eb78da8dd53632305b7b46e7ce9224d376e786f4d9c6361a4c65201c804642343ee4933f6a37d8f230d25e497858b2a762c750d9b71911cc81cd8d447af1d7d384833af2cf8211ac7d6dc4b05f8b4219b6779f3e89d047d1072c8c583e36cafbb7d8151f915d1f214952244834adb7cf57459c7f14af22d01437ce43c677279de3bb3f7d40ee64b49823294ba1f30530d885cda6864edfc4e27cd2ccb101e3ca1e44f43de80cb7377595deea1aec72&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:49:08.184696933+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064438Z&X-Goog-Expires=86399&X-Goog-Signature=1c4d8e69c4a4bc16a1ecff0d2fc584e4c3edf928e99bc626ec3d49efbad4153386ec6f02ecfb9a713d077b0e7533a5e727b8841e1f183aaa50bc27bdb38788135973a93bd004290294d6805b7dead160711dcde78af1548be3f499ef000c710b6088b024dc1f374aa65daae277aa91ff8acb08d522905678e0f093d5e15a6a5d353ff3b07e46494349753f29051a19a8b88b453cf21b66006a30a00a1c086288b991685b1aa294e714697f4eee27b2175e9fcfb3c0e19cf2c191a0414c7d7c4d8d3e27a54fa2b509a430977dfc12087aa7109df8e6f999dc0c21b1f79d1973e5f6e496b36fecdef5039685395b130a02c153b53cde1288da4a2618f61b8f6f03&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:58434->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064438Z&X-Goog-Expires=86399&X-Goog-Signature=1c4d8e69c4a4bc16a1ecff0d2fc584e4c3edf928e99bc626ec3d49efbad4153386ec6f02ecfb9a713d077b0e7533a5e727b8841e1f183aaa50bc27bdb38788135973a93bd004290294d6805b7dead160711dcde78af1548be3f499ef000c710b6088b024dc1f374aa65daae277aa91ff8acb08d522905678e0f093d5e15a6a5d353ff3b07e46494349753f29051a19a8b88b453cf21b66006a30a00a1c086288b991685b1aa294e714697f4eee27b2175e9fcfb3c0e19cf2c191a0414c7d7c4d8d3e27a54fa2b509a430977dfc12087aa7109df8e6f999dc0c21b1f79d1973e5f6e496b36fecdef5039685395b130a02c153b53cde1288da4a2618f61b8f6f03&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:49:08.238590922+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064442Z&X-Goog-Expires=86399&X-Goog-Signature=18af461e43ed41e2c3a450ebab1acbb283be2f2273ca1ff2f8566f3de8aa061a57c9a976e1e832eb09904aa60e52c339d76234f48911b7414b9ccfb9d7e65ba092532fe77282e9c592d34435b6d111b8d91f81c1283fe85304b69972c3d5258038dcc9c06748efe8517ac504ccaca2aa5d3d83a205076eebb74f2a272e8dcc12cc93743bff75472b91f9d2be2716932a5ff40617c584c06bc4f2e9c267b0a1ad8dae6823f68c45a7f7bb22e8f84eab16df5228b51c5b6daf47084b2ebca10027f9117836dbfc8f62e2d96e159fac46bb346cbbeecaf4020adc3663d6c828828fbedebde155686bb7e2e7af86fdac8373bc807cbb1b0e53f0f2fd98e3d1a33643&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:58410->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064442Z&X-Goog-Expires=86399&X-Goog-Signature=18af461e43ed41e2c3a450ebab1acbb283be2f2273ca1ff2f8566f3de8aa061a57c9a976e1e832eb09904aa60e52c339d76234f48911b7414b9ccfb9d7e65ba092532fe77282e9c592d34435b6d111b8d91f81c1283fe85304b69972c3d5258038dcc9c06748efe8517ac504ccaca2aa5d3d83a205076eebb74f2a272e8dcc12cc93743bff75472b91f9d2be2716932a5ff40617c584c06bc4f2e9c267b0a1ad8dae6823f68c45a7f7bb22e8f84eab16df5228b51c5b6daf47084b2ebca10027f9117836dbfc8f62e2d96e159fac46bb346cbbeecaf4020adc3663d6c828828fbedebde155686bb7e2e7af86fdac8373bc807cbb1b0e53f0f2fd98e3d1a33643&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:49:13.157250807+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064442Z&X-Goog-Expires=86399&X-Goog-Signature=18af461e43ed41e2c3a450ebab1acbb283be2f2273ca1ff2f8566f3de8aa061a57c9a976e1e832eb09904aa60e52c339d76234f48911b7414b9ccfb9d7e65ba092532fe77282e9c592d34435b6d111b8d91f81c1283fe85304b69972c3d5258038dcc9c06748efe8517ac504ccaca2aa5d3d83a205076eebb74f2a272e8dcc12cc93743bff75472b91f9d2be2716932a5ff40617c584c06bc4f2e9c267b0a1ad8dae6823f68c45a7f7bb22e8f84eab16df5228b51c5b6daf47084b2ebca10027f9117836dbfc8f62e2d96e159fac46bb346cbbeecaf4020adc3663d6c828828fbedebde155686bb7e2e7af86fdac8373bc807cbb1b0e53f0f2fd98e3d1a33643&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:58436->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064442Z&X-Goog-Expires=86399&X-Goog-Signature=18af461e43ed41e2c3a450ebab1acbb283be2f2273ca1ff2f8566f3de8aa061a57c9a976e1e832eb09904aa60e52c339d76234f48911b7414b9ccfb9d7e65ba092532fe77282e9c592d34435b6d111b8d91f81c1283fe85304b69972c3d5258038dcc9c06748efe8517ac504ccaca2aa5d3d83a205076eebb74f2a272e8dcc12cc93743bff75472b91f9d2be2716932a5ff40617c584c06bc4f2e9c267b0a1ad8dae6823f68c45a7f7bb22e8f84eab16df5228b51c5b6daf47084b2ebca10027f9117836dbfc8f62e2d96e159fac46bb346cbbeecaf4020adc3663d6c828828fbedebde155686bb7e2e7af86fdac8373bc807cbb1b0e53f0f2fd98e3d1a33643&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:49:18.001715505+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064438Z&X-Goog-Expires=86399&X-Goog-Signature=1c4d8e69c4a4bc16a1ecff0d2fc584e4c3edf928e99bc626ec3d49efbad4153386ec6f02ecfb9a713d077b0e7533a5e727b8841e1f183aaa50bc27bdb38788135973a93bd004290294d6805b7dead160711dcde78af1548be3f499ef000c710b6088b024dc1f374aa65daae277aa91ff8acb08d522905678e0f093d5e15a6a5d353ff3b07e46494349753f29051a19a8b88b453cf21b66006a30a00a1c086288b991685b1aa294e714697f4eee27b2175e9fcfb3c0e19cf2c191a0414c7d7c4d8d3e27a54fa2b509a430977dfc12087aa7109df8e6f999dc0c21b1f79d1973e5f6e496b36fecdef5039685395b130a02c153b53cde1288da4a2618f61b8f6f03&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:36496->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064438Z&X-Goog-Expires=86399&X-Goog-Signature=1c4d8e69c4a4bc16a1ecff0d2fc584e4c3edf928e99bc626ec3d49efbad4153386ec6f02ecfb9a713d077b0e7533a5e727b8841e1f183aaa50bc27bdb38788135973a93bd004290294d6805b7dead160711dcde78af1548be3f499ef000c710b6088b024dc1f374aa65daae277aa91ff8acb08d522905678e0f093d5e15a6a5d353ff3b07e46494349753f29051a19a8b88b453cf21b66006a30a00a1c086288b991685b1aa294e714697f4eee27b2175e9fcfb3c0e19cf2c191a0414c7d7c4d8d3e27a54fa2b509a430977dfc12087aa7109df8e6f999dc0c21b1f79d1973e5f6e496b36fecdef5039685395b130a02c153b53cde1288da4a2618f61b8f6f03&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:51:14.898455084+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064442Z&X-Goog-Expires=86399&X-Goog-Signature=552d74aaa0c7a9ee8b16a055c4a3ae605223c81742d6231450f19f617797cdf1d95d8c626b624b1ae60476efdea6a6493422a74e44cfb29852257c4d29f9af812376ffea925f7202ef9d4e6d8b45406df628af59698be20fd994c7fc8357d6382e5db66115fd4506f8d3cc341450e297ca450ab1415621427e7e038e8298cf4693042ab314dda6c11ecacb5c31b9149ee121274a41ebf872a03f5d03bd65418bca7c8793eb46f19239ca0e2b83c5e22aeedb4ab15f310d51023c98305115916c47bc12887198169563b297c4539e2921fe7fadc5c61a3c195e7cc9420a952f614cbce034fb3e0a9c025e35797c41d4d91065e34db90fa48d47fab4198a2a9945&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:48578->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064442Z&X-Goog-Expires=86399&X-Goog-Signature=552d74aaa0c7a9ee8b16a055c4a3ae605223c81742d6231450f19f617797cdf1d95d8c626b624b1ae60476efdea6a6493422a74e44cfb29852257c4d29f9af812376ffea925f7202ef9d4e6d8b45406df628af59698be20fd994c7fc8357d6382e5db66115fd4506f8d3cc341450e297ca450ab1415621427e7e038e8298cf4693042ab314dda6c11ecacb5c31b9149ee121274a41ebf872a03f5d03bd65418bca7c8793eb46f19239ca0e2b83c5e22aeedb4ab15f310d51023c98305115916c47bc12887198169563b297c4539e2921fe7fadc5c61a3c195e7cc9420a952f614cbce034fb3e0a9c025e35797c41d4d91065e34db90fa48d47fab4198a2a9945&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:51:31.292954507+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064442Z&X-Goog-Expires=86399&X-Goog-Signature=18af461e43ed41e2c3a450ebab1acbb283be2f2273ca1ff2f8566f3de8aa061a57c9a976e1e832eb09904aa60e52c339d76234f48911b7414b9ccfb9d7e65ba092532fe77282e9c592d34435b6d111b8d91f81c1283fe85304b69972c3d5258038dcc9c06748efe8517ac504ccaca2aa5d3d83a205076eebb74f2a272e8dcc12cc93743bff75472b91f9d2be2716932a5ff40617c584c06bc4f2e9c267b0a1ad8dae6823f68c45a7f7bb22e8f84eab16df5228b51c5b6daf47084b2ebca10027f9117836dbfc8f62e2d96e159fac46bb346cbbeecaf4020adc3663d6c828828fbedebde155686bb7e2e7af86fdac8373bc807cbb1b0e53f0f2fd98e3d1a33643&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:49362->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064442Z&X-Goog-Expires=86399&X-Goog-Signature=18af461e43ed41e2c3a450ebab1acbb283be2f2273ca1ff2f8566f3de8aa061a57c9a976e1e832eb09904aa60e52c339d76234f48911b7414b9ccfb9d7e65ba092532fe77282e9c592d34435b6d111b8d91f81c1283fe85304b69972c3d5258038dcc9c06748efe8517ac504ccaca2aa5d3d83a205076eebb74f2a272e8dcc12cc93743bff75472b91f9d2be2716932a5ff40617c584c06bc4f2e9c267b0a1ad8dae6823f68c45a7f7bb22e8f84eab16df5228b51c5b6daf47084b2ebca10027f9117836dbfc8f62e2d96e159fac46bb346cbbeecaf4020adc3663d6c828828fbedebde155686bb7e2e7af86fdac8373bc807cbb1b0e53f0f2fd98e3d1a33643&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:51:48.908372115+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064442Z&X-Goog-Expires=86399&X-Goog-Signature=18af461e43ed41e2c3a450ebab1acbb283be2f2273ca1ff2f8566f3de8aa061a57c9a976e1e832eb09904aa60e52c339d76234f48911b7414b9ccfb9d7e65ba092532fe77282e9c592d34435b6d111b8d91f81c1283fe85304b69972c3d5258038dcc9c06748efe8517ac504ccaca2aa5d3d83a205076eebb74f2a272e8dcc12cc93743bff75472b91f9d2be2716932a5ff40617c584c06bc4f2e9c267b0a1ad8dae6823f68c45a7f7bb22e8f84eab16df5228b51c5b6daf47084b2ebca10027f9117836dbfc8f62e2d96e159fac46bb346cbbeecaf4020adc3663d6c828828fbedebde155686bb7e2e7af86fdac8373bc807cbb1b0e53f0f2fd98e3d1a33643&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:38778->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064442Z&X-Goog-Expires=86399&X-Goog-Signature=18af461e43ed41e2c3a450ebab1acbb283be2f2273ca1ff2f8566f3de8aa061a57c9a976e1e832eb09904aa60e52c339d76234f48911b7414b9ccfb9d7e65ba092532fe77282e9c592d34435b6d111b8d91f81c1283fe85304b69972c3d5258038dcc9c06748efe8517ac504ccaca2aa5d3d83a205076eebb74f2a272e8dcc12cc93743bff75472b91f9d2be2716932a5ff40617c584c06bc4f2e9c267b0a1ad8dae6823f68c45a7f7bb22e8f84eab16df5228b51c5b6daf47084b2ebca10027f9117836dbfc8f62e2d96e159fac46bb346cbbeecaf4020adc3663d6c828828fbedebde155686bb7e2e7af86fdac8373bc807cbb1b0e53f0f2fd98e3d1a33643&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:52:22.529468861+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064442Z&X-Goog-Expires=86399&X-Goog-Signature=18af461e43ed41e2c3a450ebab1acbb283be2f2273ca1ff2f8566f3de8aa061a57c9a976e1e832eb09904aa60e52c339d76234f48911b7414b9ccfb9d7e65ba092532fe77282e9c592d34435b6d111b8d91f81c1283fe85304b69972c3d5258038dcc9c06748efe8517ac504ccaca2aa5d3d83a205076eebb74f2a272e8dcc12cc93743bff75472b91f9d2be2716932a5ff40617c584c06bc4f2e9c267b0a1ad8dae6823f68c45a7f7bb22e8f84eab16df5228b51c5b6daf47084b2ebca10027f9117836dbfc8f62e2d96e159fac46bb346cbbeecaf4020adc3663d6c828828fbedebde155686bb7e2e7af86fdac8373bc807cbb1b0e53f0f2fd98e3d1a33643&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:49956->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064442Z&X-Goog-Expires=86399&X-Goog-Signature=18af461e43ed41e2c3a450ebab1acbb283be2f2273ca1ff2f8566f3de8aa061a57c9a976e1e832eb09904aa60e52c339d76234f48911b7414b9ccfb9d7e65ba092532fe77282e9c592d34435b6d111b8d91f81c1283fe85304b69972c3d5258038dcc9c06748efe8517ac504ccaca2aa5d3d83a205076eebb74f2a272e8dcc12cc93743bff75472b91f9d2be2716932a5ff40617c584c06bc4f2e9c267b0a1ad8dae6823f68c45a7f7bb22e8f84eab16df5228b51c5b6daf47084b2ebca10027f9117836dbfc8f62e2d96e159fac46bb346cbbeecaf4020adc3663d6c828828fbedebde155686bb7e2e7af86fdac8373bc807cbb1b0e53f0f2fd98e3d1a33643&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:52:28.74584019+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064443Z&X-Goog-Expires=86399&X-Goog-Signature=a1ed9a3895a7355448222045946c9063c3c6f485bd42fdbf0f75878011690b13c231c0d4aca1dc09a5e1858884868f833ed37fe1c60079b13bd9b6bfc8819178c67a00fab449892d0eb5d54823719610deefa2051881eeed0f85c20a794ae5fbc58c588cb260634bf12a6e9adabcb55dcaa39014e44df8f6e83cdb24b4e514a211d9981878a93b681e87792d5e3d8c92786a6ba10ab50a7731f91112fbdddef9aad498d4c0513b08d85241958614cbd19074b46becfe64edc60792f69a44d99612911006d970e3514eb0813dd8af1b603c91053990e61f1ef863736debc5dac74879bbed807d2e917ea1fef6257a1be0c86b9abda3db595ba8f0776d7f59639b&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:55480->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064443Z&X-Goog-Expires=86399&X-Goog-Signature=a1ed9a3895a7355448222045946c9063c3c6f485bd42fdbf0f75878011690b13c231c0d4aca1dc09a5e1858884868f833ed37fe1c60079b13bd9b6bfc8819178c67a00fab449892d0eb5d54823719610deefa2051881eeed0f85c20a794ae5fbc58c588cb260634bf12a6e9adabcb55dcaa39014e44df8f6e83cdb24b4e514a211d9981878a93b681e87792d5e3d8c92786a6ba10ab50a7731f91112fbdddef9aad498d4c0513b08d85241958614cbd19074b46becfe64edc60792f69a44d99612911006d970e3514eb0813dd8af1b603c91053990e61f1ef863736debc5dac74879bbed807d2e917ea1fef6257a1be0c86b9abda3db595ba8f0776d7f59639b&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:53:29.135553591+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064443Z&X-Goog-Expires=86399&X-Goog-Signature=a1ed9a3895a7355448222045946c9063c3c6f485bd42fdbf0f75878011690b13c231c0d4aca1dc09a5e1858884868f833ed37fe1c60079b13bd9b6bfc8819178c67a00fab449892d0eb5d54823719610deefa2051881eeed0f85c20a794ae5fbc58c588cb260634bf12a6e9adabcb55dcaa39014e44df8f6e83cdb24b4e514a211d9981878a93b681e87792d5e3d8c92786a6ba10ab50a7731f91112fbdddef9aad498d4c0513b08d85241958614cbd19074b46becfe64edc60792f69a44d99612911006d970e3514eb0813dd8af1b603c91053990e61f1ef863736debc5dac74879bbed807d2e917ea1fef6257a1be0c86b9abda3db595ba8f0776d7f59639b&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:48996->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064443Z&X-Goog-Expires=86399&X-Goog-Signature=a1ed9a3895a7355448222045946c9063c3c6f485bd42fdbf0f75878011690b13c231c0d4aca1dc09a5e1858884868f833ed37fe1c60079b13bd9b6bfc8819178c67a00fab449892d0eb5d54823719610deefa2051881eeed0f85c20a794ae5fbc58c588cb260634bf12a6e9adabcb55dcaa39014e44df8f6e83cdb24b4e514a211d9981878a93b681e87792d5e3d8c92786a6ba10ab50a7731f91112fbdddef9aad498d4c0513b08d85241958614cbd19074b46becfe64edc60792f69a44d99612911006d970e3514eb0813dd8af1b603c91053990e61f1ef863736debc5dac74879bbed807d2e917ea1fef6257a1be0c86b9abda3db595ba8f0776d7f59639b&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:53:38.183430588+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064438Z&X-Goog-Expires=86399&X-Goog-Signature=b784e18b1e51706c965d5e39b31f26a5ab282191d4e34eba448ded3882706352d7cd0eb8afc78f4f6585ac2292839aa84719202f49c853ee4b4d5930137d15a5ca9c3bf336b9196fa43ea17730c012e474eb78da8dd53632305b7b46e7ce9224d376e786f4d9c6361a4c65201c804642343ee4933f6a37d8f230d25e497858b2a762c750d9b71911cc81cd8d447af1d7d384833af2cf8211ac7d6dc4b05f8b4219b6779f3e89d047d1072c8c583e36cafbb7d8151f915d1f214952244834adb7cf57459c7f14af22d01437ce43c677279de3bb3f7d40ee64b49823294ba1f30530d885cda6864edfc4e27cd2ccb101e3ca1e44f43de80cb7377595deea1aec72&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:45444->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064438Z&X-Goog-Expires=86399&X-Goog-Signature=b784e18b1e51706c965d5e39b31f26a5ab282191d4e34eba448ded3882706352d7cd0eb8afc78f4f6585ac2292839aa84719202f49c853ee4b4d5930137d15a5ca9c3bf336b9196fa43ea17730c012e474eb78da8dd53632305b7b46e7ce9224d376e786f4d9c6361a4c65201c804642343ee4933f6a37d8f230d25e497858b2a762c750d9b71911cc81cd8d447af1d7d384833af2cf8211ac7d6dc4b05f8b4219b6779f3e89d047d1072c8c583e36cafbb7d8151f915d1f214952244834adb7cf57459c7f14af22d01437ce43c677279de3bb3f7d40ee64b49823294ba1f30530d885cda6864edfc4e27cd2ccb101e3ca1e44f43de80cb7377595deea1aec72&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:53:58.779409183+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064438Z&X-Goog-Expires=86399&X-Goog-Signature=1c4d8e69c4a4bc16a1ecff0d2fc584e4c3edf928e99bc626ec3d49efbad4153386ec6f02ecfb9a713d077b0e7533a5e727b8841e1f183aaa50bc27bdb38788135973a93bd004290294d6805b7dead160711dcde78af1548be3f499ef000c710b6088b024dc1f374aa65daae277aa91ff8acb08d522905678e0f093d5e15a6a5d353ff3b07e46494349753f29051a19a8b88b453cf21b66006a30a00a1c086288b991685b1aa294e714697f4eee27b2175e9fcfb3c0e19cf2c191a0414c7d7c4d8d3e27a54fa2b509a430977dfc12087aa7109df8e6f999dc0c21b1f79d1973e5f6e496b36fecdef5039685395b130a02c153b53cde1288da4a2618f61b8f6f03&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:52344->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064438Z&X-Goog-Expires=86399&X-Goog-Signature=1c4d8e69c4a4bc16a1ecff0d2fc584e4c3edf928e99bc626ec3d49efbad4153386ec6f02ecfb9a713d077b0e7533a5e727b8841e1f183aaa50bc27bdb38788135973a93bd004290294d6805b7dead160711dcde78af1548be3f499ef000c710b6088b024dc1f374aa65daae277aa91ff8acb08d522905678e0f093d5e15a6a5d353ff3b07e46494349753f29051a19a8b88b453cf21b66006a30a00a1c086288b991685b1aa294e714697f4eee27b2175e9fcfb3c0e19cf2c191a0414c7d7c4d8d3e27a54fa2b509a430977dfc12087aa7109df8e6f999dc0c21b1f79d1973e5f6e496b36fecdef5039685395b130a02c153b53cde1288da4a2618f61b8f6f03&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:55:32.951218418+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064442Z&X-Goog-Expires=86399&X-Goog-Signature=18af461e43ed41e2c3a450ebab1acbb283be2f2273ca1ff2f8566f3de8aa061a57c9a976e1e832eb09904aa60e52c339d76234f48911b7414b9ccfb9d7e65ba092532fe77282e9c592d34435b6d111b8d91f81c1283fe85304b69972c3d5258038dcc9c06748efe8517ac504ccaca2aa5d3d83a205076eebb74f2a272e8dcc12cc93743bff75472b91f9d2be2716932a5ff40617c584c06bc4f2e9c267b0a1ad8dae6823f68c45a7f7bb22e8f84eab16df5228b51c5b6daf47084b2ebca10027f9117836dbfc8f62e2d96e159fac46bb346cbbeecaf4020adc3663d6c828828fbedebde155686bb7e2e7af86fdac8373bc807cbb1b0e53f0f2fd98e3d1a33643&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:33980->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064442Z&X-Goog-Expires=86399&X-Goog-Signature=18af461e43ed41e2c3a450ebab1acbb283be2f2273ca1ff2f8566f3de8aa061a57c9a976e1e832eb09904aa60e52c339d76234f48911b7414b9ccfb9d7e65ba092532fe77282e9c592d34435b6d111b8d91f81c1283fe85304b69972c3d5258038dcc9c06748efe8517ac504ccaca2aa5d3d83a205076eebb74f2a272e8dcc12cc93743bff75472b91f9d2be2716932a5ff40617c584c06bc4f2e9c267b0a1ad8dae6823f68c45a7f7bb22e8f84eab16df5228b51c5b6daf47084b2ebca10027f9117836dbfc8f62e2d96e159fac46bb346cbbeecaf4020adc3663d6c828828fbedebde155686bb7e2e7af86fdac8373bc807cbb1b0e53f0f2fd98e3d1a33643&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:55:41.170491233+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064442Z&X-Goog-Expires=86399&X-Goog-Signature=552d74aaa0c7a9ee8b16a055c4a3ae605223c81742d6231450f19f617797cdf1d95d8c626b624b1ae60476efdea6a6493422a74e44cfb29852257c4d29f9af812376ffea925f7202ef9d4e6d8b45406df628af59698be20fd994c7fc8357d6382e5db66115fd4506f8d3cc341450e297ca450ab1415621427e7e038e8298cf4693042ab314dda6c11ecacb5c31b9149ee121274a41ebf872a03f5d03bd65418bca7c8793eb46f19239ca0e2b83c5e22aeedb4ab15f310d51023c98305115916c47bc12887198169563b297c4539e2921fe7fadc5c61a3c195e7cc9420a952f614cbce034fb3e0a9c025e35797c41d4d91065e34db90fa48d47fab4198a2a9945&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:52256->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/b7u1a6df/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T064442Z&X-Goog-Expires=86399&X-Goog-Signature=552d74aaa0c7a9ee8b16a055c4a3ae605223c81742d6231450f19f617797cdf1d95d8c626b624b1ae60476efdea6a6493422a74e44cfb29852257c4d29f9af812376ffea925f7202ef9d4e6d8b45406df628af59698be20fd994c7fc8357d6382e5db66115fd4506f8d3cc341450e297ca450ab1415621427e7e038e8298cf4693042ab314dda6c11ecacb5c31b9149ee121274a41ebf872a03f5d03bd65418bca7c8793eb46f19239ca0e2b83c5e22aeedb4ab15f310d51023c98305115916c47bc12887198169563b297c4539e2921fe7fadc5c61a3c195e7cc9420a952f614cbce034fb3e0a9c025e35797c41d4d91065e34db90fa48d47fab4198a2a9945&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} diff --git a/BioReason_new/wandb/run-20250813_144434-b7u1a6df/logs/debug.log b/BioReason_new/wandb/run-20250813_144434-b7u1a6df/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..ac7f886e495349bc3d7d17025f2def602cd0fa4b --- /dev/null +++ b/BioReason_new/wandb/run-20250813_144434-b7u1a6df/logs/debug.log @@ -0,0 +1,6780 @@ +2025-08-13 14:44:34,957 INFO MainThread:9283 [wandb_setup.py:_flush():80] Current SDK version is 0.21.1 +2025-08-13 14:44:34,957 INFO MainThread:9283 [wandb_setup.py:_flush():80] Configure stats pid to 9283 +2025-08-13 14:44:34,957 INFO MainThread:9283 [wandb_setup.py:_flush():80] Loading settings from /root/.config/wandb/settings +2025-08-13 14:44:34,957 INFO MainThread:9283 [wandb_setup.py:_flush():80] Loading settings from /nas/shared/kilab/wangyujia/BioReason_new/wandb/settings +2025-08-13 14:44:34,957 INFO MainThread:9283 [wandb_setup.py:_flush():80] Loading settings from environment variables +2025-08-13 14:44:34,957 INFO MainThread:9283 [wandb_init.py:setup_run_log_directory():703] Logging user logs to /nas/shared/kilab/wangyujia/BioReason_new/wandb/run-20250813_144434-b7u1a6df/logs/debug.log +2025-08-13 14:44:34,957 INFO MainThread:9283 [wandb_init.py:setup_run_log_directory():704] Logging internal logs to /nas/shared/kilab/wangyujia/BioReason_new/wandb/run-20250813_144434-b7u1a6df/logs/debug-internal.log +2025-08-13 14:44:34,957 INFO MainThread:9283 [wandb_init.py:init():830] calling init triggers +2025-08-13 14:44:34,957 INFO MainThread:9283 [wandb_init.py:init():835] wandb.init called with sweep_config: {} +config: {'text_model_name': '/oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged', 'protein_model_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m', 'qformer_model_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft', 'cache_dir': '/model-weights', 'num_query_tokens': 8, 'train_dataset': '/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl', 'valid_dataset': '/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl', 'eval_dataset': True, 'output_dir': './contrastive_outputs', 'num_epochs': 10, 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 0.01, 'warmup_steps': 1000, 'gradient_accumulation_steps': 1, 'temperature': 0.07, 'freeze_protein_model': True, 'freeze_text_model': True, 'protein_weight': 1.0, 'text_weight': 1.0, 'enable_ptm': True, 'ptm_weight': 1.0, 'max_length_protein': 1024, 'max_length_text': 512, 'num_workers': 8, 'logging_steps': 100, 'eval_steps': 500, 'save_steps': 1000, 'save_total_limit': 3, 'fp16': False, 'bf16': False, 'seed': 42, 'use_wandb': True, 'wandb_project': 'protein-llm-contrastive', 'wandb_entity': None, '_wandb': {}} +2025-08-13 14:44:34,957 INFO MainThread:9283 [wandb_init.py:init():871] starting backend +2025-08-13 14:44:35,165 INFO MainThread:9283 [wandb_init.py:init():874] sending inform_init request +2025-08-13 14:44:35,171 INFO MainThread:9283 [wandb_init.py:init():882] backend started and connected +2025-08-13 14:44:35,172 INFO MainThread:9283 [wandb_init.py:init():953] updated telemetry +2025-08-13 14:44:35,200 INFO MainThread:9283 [wandb_init.py:init():977] communicating run to backend with 90.0 second timeout +2025-08-13 14:44:38,250 INFO MainThread:9283 [wandb_init.py:init():1029] starting run threads in backend +2025-08-13 14:44:38,368 INFO MainThread:9283 [wandb_run.py:_console_start():2494] atexit reg +2025-08-13 14:44:38,368 INFO MainThread:9283 [wandb_run.py:_redirect():2342] redirect: wrap_raw +2025-08-13 14:44:38,368 INFO MainThread:9283 [wandb_run.py:_redirect():2411] Wrapping output streams. +2025-08-13 14:44:38,368 INFO MainThread:9283 [wandb_run.py:_redirect():2434] Redirects installed. +2025-08-13 14:44:38,370 INFO MainThread:9283 [wandb_init.py:init():1075] run started, returning control to user process +2025-08-13 14:44:41,989 INFO MsgRouterThr:9283 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles. +2025-08-13 14:56:13,872 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,893 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,894 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,894 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,895 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,895 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,896 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,896 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,897 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,897 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,897 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,898 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,898 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,899 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,899 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,899 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,900 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,900 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,900 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,901 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,901 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,902 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,902 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,902 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,903 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,903 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,903 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,904 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,904 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,905 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,905 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,905 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,906 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,906 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,906 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,907 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,907 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,907 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,908 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,908 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,909 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,909 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,909 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,910 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,910 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,911 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,911 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,911 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,912 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,912 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,912 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,913 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,913 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,914 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,914 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,914 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,915 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,915 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,915 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,916 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,916 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,916 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,917 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,917 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,918 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,918 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,918 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,919 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,919 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,919 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,920 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,920 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,920 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,921 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,921 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,922 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,922 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,922 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,923 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,923 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,924 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,924 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,924 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,925 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,925 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,925 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,926 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,926 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,927 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,927 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,927 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,928 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,928 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,928 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,929 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,929 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,929 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,930 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,930 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,930 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,931 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,931 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,932 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,932 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,932 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,933 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,933 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,933 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,934 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,934 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,934 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,935 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,935 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,935 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,936 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,936 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,937 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,937 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,937 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,938 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,938 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,938 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,939 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,939 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,940 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,940 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,940 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,941 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,941 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,942 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,942 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,942 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,943 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,943 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,943 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,944 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,944 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,944 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,945 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,945 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,945 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,946 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,946 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,947 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,947 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,947 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,948 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,948 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,948 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,949 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,949 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,949 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,950 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,950 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,950 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,951 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,951 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,951 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,952 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,952 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,953 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,953 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,954 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,954 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,954 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,955 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,955 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,955 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,956 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,956 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,956 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,957 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,957 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,958 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,958 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,958 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,959 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,959 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,959 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,960 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,960 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,960 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,961 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,961 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,961 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,962 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,962 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,962 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,963 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,963 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,964 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,964 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,964 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,965 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,965 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,965 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,966 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,966 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,966 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,967 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,967 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,967 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,968 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,968 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,969 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,969 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,969 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,970 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,970 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,970 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,971 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,971 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,971 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,972 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,972 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,972 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,973 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-08-13 14:56:13,973 ERROR MainThread:9283 [redirect.py:_on_write():664] [all runs] error in stderr callback +Traceback (most recent call last): + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write + cb(written_data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2421, in + lambda data: self._console_raw_callback("stderr", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 399, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 457, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1583, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 771, in publish_output_raw + self._publish_output_raw(o) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 166, in send_record_publish + self.send_server_request(server_req) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 146, in send_server_request + self._send_message(msg) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 143, in _send_message + self._sendall_with_error_handle(header + data) + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 122, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe diff --git a/BioReason_new/wandb/run-20250813_144434-b7u1a6df/run-b7u1a6df.wandb b/BioReason_new/wandb/run-20250813_144434-b7u1a6df/run-b7u1a6df.wandb new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/BioReason_new/wandb/run-20250813_145627-ep0ib6n2/files/config.yaml b/BioReason_new/wandb/run-20250813_145627-ep0ib6n2/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..79533411de7db45f01fe23423e8f46a409e579db --- /dev/null +++ b/BioReason_new/wandb/run-20250813_145627-ep0ib6n2/files/config.yaml @@ -0,0 +1,458 @@ +_wandb: + value: + cli_version: 0.21.1 + e: + 8kepep2ysih7kn77h1iftipfg3shqy5q: + args: + - --text_model_name + - /oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged + - --protein_model_name + - /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m + - --qformer_model_name + - /nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft + - --num_query_tokens + - "8" + - --train_dataset + - /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl + - --valid_dataset + - /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl + - --output_dir + - ./contrastive_outputs + - --num_epochs + - "10" + - --batch_size + - "8" + - --learning_rate + - "1e-4" + - --temperature + - "0.07" + - --freeze_protein_model + - --freeze_text_model + - --enable_ptm + - --max_length_protein + - "1024" + - --max_length_text + - "512" + - --num_workers + - "8" + - --eval_dataset + - --use_wandb + - --wandb_project + - protein-llm-contrastive + - --logging_steps + - "100" + - --eval_steps + - "500" + - --save_steps + - "1000" + codePath: wangyujia/BioReason_new/train_contrastive.py + codePathLocal: train_contrastive.py + cpu_count: 64 + cpu_count_logical: 64 + cudaVersion: "12.1" + disk: + /: + total: "1623302262784" + used: "1015189504" + email: gia0603yucca@gmail.com + executable: /root/miniconda3/envs/bioreason/bin/python + git: + commit: b8caf406aa1699c788f0ca6e44a1769452c317db + remote: https://github.com/PorUna-byte/PAR.git + gpu: NVIDIA A800-SXM4-80GB + gpu_count: 8 + gpu_nvidia: + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-62fcc532-880f-7324-eb13-3b739a38c29b + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-af1197aa-3e7d-a53e-185d-04b1d64164da + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-3e6e551f-2d7d-a27b-654a-da56933d2283 + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-a947b64a-c545-ee5e-30e6-b4b0e2dd8fdc + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-ebd04f3f-c437-1b8b-26f2-4b1ebfee93c4 + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-53994891-72a6-25be-a4c8-e32c695155b0 + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-a927d00a-15c5-31f5-9ae1-9d2e773821a1 + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-93d555c4-29a0-cd78-abac-d98cd160d4e1 + host: dsw-265304-585cc9d768-ckfd7 + memory: + total: "549755813888" + os: Linux-5.10.134-008.18.kangaroo.al8.x86_64-x86_64-with-glibc2.35 + program: /nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py + python: CPython 3.11.0 + root: /nas/shared/kilab/wangyujia/BioReason_new + startedAt: "2025-08-13T06:56:27.037635Z" + writerId: 8kepep2ysih7kn77h1iftipfg3shqy5q + m: + - "1": train/global_step + "6": + - 3 + "7": [] + - "2": '*' + "5": 1 + "6": + - 1 + "7": [] + python_version: 3.11.0 + t: + "1": + - 1 + - 9 + - 11 + - 41 + - 49 + - 51 + - 71 + - 84 + - 98 + - 103 + "2": + - 1 + - 9 + - 11 + - 41 + - 49 + - 51 + - 71 + - 84 + - 98 + - 103 + "3": + - 7 + - 13 + - 16 + - 66 + "4": 3.11.0 + "5": 0.21.1 + "6": 4.55.0 + "9": + "1": transformers_trainer + "12": 0.21.1 + "13": linux-x86_64 +accelerator_config: + value: + dispatch_batches: null + even_batches: true + gradient_accumulation_kwargs: null + non_blocking: false + split_batches: false + use_seedable_sampler: true +adafactor: + value: false +adam_beta1: + value: 0.9 +adam_beta2: + value: 0.999 +adam_epsilon: + value: 1e-08 +auto_find_batch_size: + value: false +average_tokens_across_devices: + value: false +batch_eval_metrics: + value: false +batch_size: + value: 8 +bf16: + value: false +bf16_full_eval: + value: false +cache_dir: + value: /model-weights +data_seed: + value: null +dataloader_drop_last: + value: false +dataloader_num_workers: + value: 8 +dataloader_persistent_workers: + value: false +dataloader_pin_memory: + value: true +dataloader_prefetch_factor: + value: null +ddp_backend: + value: null +ddp_broadcast_buffers: + value: null +ddp_bucket_cap_mb: + value: null +ddp_find_unused_parameters: + value: false +ddp_timeout: + value: 1800 +debug: + value: [] +deepspeed: + value: null +disable_tqdm: + value: false +do_eval: + value: true +do_predict: + value: false +do_train: + value: false +enable_ptm: + value: true +eval_accumulation_steps: + value: null +eval_dataset: + value: true +eval_delay: + value: 0 +eval_do_concat_batches: + value: true +eval_on_start: + value: false +eval_steps: + value: 500 +eval_strategy: + value: steps +eval_use_gather_object: + value: false +fp16: + value: false +fp16_backend: + value: auto +fp16_full_eval: + value: false +fp16_opt_level: + value: O1 +freeze_protein_model: + value: true +freeze_text_model: + value: true +fsdp: + value: [] +fsdp_config: + value: + min_num_params: 0 + xla: false + xla_fsdp_grad_ckpt: false + xla_fsdp_v2: false +fsdp_min_num_params: + value: 0 +fsdp_transformer_layer_cls_to_wrap: + value: null +full_determinism: + value: false +gradient_accumulation_steps: + value: 1 +gradient_checkpointing: + value: false +gradient_checkpointing_kwargs: + value: null +greater_is_better: + value: true +group_by_length: + value: false +half_precision_backend: + value: auto +hub_always_push: + value: false +hub_model_id: + value: null +hub_private_repo: + value: null +hub_revision: + value: null +hub_strategy: + value: every_save +hub_token: + value: +ignore_data_skip: + value: false +include_for_metrics: + value: [] +include_inputs_for_metrics: + value: false +include_num_input_tokens_seen: + value: false +include_tokens_per_second: + value: false +jit_mode_eval: + value: false +label_names: + value: null +label_smoothing_factor: + value: 0 +learning_rate: + value: 0.0001 +length_column_name: + value: length +liger_kernel_config: + value: null +load_best_model_at_end: + value: true +local_rank: + value: 0 +log_level: + value: passive +log_level_replica: + value: warning +log_on_each_node: + value: true +logging_dir: + value: ./contrastive_outputs/runs/Aug13_14-56-41_dsw-265304-585cc9d768-ckfd7 +logging_first_step: + value: false +logging_nan_inf_filter: + value: true +logging_steps: + value: 100 +logging_strategy: + value: steps +lr_scheduler_type: + value: linear +max_grad_norm: + value: 1 +max_length_protein: + value: 1024 +max_length_text: + value: 512 +max_steps: + value: -1 +metric_for_best_model: + value: eval_avg_recall_at_1 +mp_parameters: + value: "" +neftune_noise_alpha: + value: null +no_cuda: + value: false +num_epochs: + value: 10 +num_query_tokens: + value: 8 +num_train_epochs: + value: 10 +num_workers: + value: 8 +optim: + value: adamw_torch +optim_args: + value: null +optim_target_modules: + value: null +output_dir: + value: ./contrastive_outputs +overwrite_output_dir: + value: false +past_index: + value: -1 +per_device_eval_batch_size: + value: 8 +per_device_train_batch_size: + value: 8 +per_gpu_eval_batch_size: + value: null +per_gpu_train_batch_size: + value: null +prediction_loss_only: + value: false +protein_model_name: + value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m +protein_weight: + value: 1 +ptm_weight: + value: 1 +push_to_hub: + value: false +push_to_hub_model_id: + value: null +push_to_hub_organization: + value: null +push_to_hub_token: + value: +qformer_model_name: + value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft +ray_scope: + value: last +remove_unused_columns: + value: false +report_to: + value: + - wandb +restore_callback_states_from_checkpoint: + value: false +resume_from_checkpoint: + value: null +run_name: + value: null +save_on_each_node: + value: false +save_only_model: + value: false +save_safetensors: + value: true +save_steps: + value: 1000 +save_strategy: + value: steps +save_total_limit: + value: 3 +seed: + value: 42 +skip_memory_metrics: + value: true +temperature: + value: 0.07 +text_model_name: + value: /oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged +text_weight: + value: 1 +tf32: + value: null +torch_compile: + value: false +torch_compile_backend: + value: null +torch_compile_mode: + value: null +torch_empty_cache_steps: + value: null +torchdynamo: + value: null +tpu_metrics_debug: + value: false +tpu_num_cores: + value: null +train_dataset: + value: /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl +use_cpu: + value: false +use_ipex: + value: false +use_legacy_prediction_loop: + value: false +use_liger_kernel: + value: false +use_mps_device: + value: false +use_wandb: + value: true +valid_dataset: + value: /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl +wandb_entity: + value: null +wandb_project: + value: protein-llm-contrastive +warmup_ratio: + value: 0 +warmup_steps: + value: 1000 +weight_decay: + value: 0.01 diff --git a/BioReason_new/wandb/run-20250813_145627-ep0ib6n2/files/output.log b/BioReason_new/wandb/run-20250813_145627-ep0ib6n2/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..c1ff7af7f8f26db79f83ab8d90f54e2071dbbb0a --- /dev/null +++ b/BioReason_new/wandb/run-20250813_145627-ep0ib6n2/files/output.log @@ -0,0 +1,39 @@ +Loading model... +Loading checkpoint shards: 100%|██████████████████████████████████████████████████████| 4/4 [00:01<00:00, 2.46it/s] +Some weights of EsmModel were not initialized from the model checkpoint at /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m and are newly initialized: ['pooler.dense.bias', 'pooler.dense.weight'] +You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference. +Loading datasets... +Training dataset size: 430595 +Eval dataset size: 10000 +Initializing enhanced trainer... +[2025-08-13 14:56:45,813] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) +/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/_distutils_hack/__init__.py:53: UserWarning: Reliance on distutils from stdlib is deprecated. Users must rely on setuptools to provide the distutils module. Avoid importing distutils or import setuptools first, and avoid setting SETUPTOOLS_USE_DISTUTILS=stdlib. Register concerns at https://github.com/pypa/setuptools/issues/new?template=distutils-deprecation.yml + warnings.warn( +[2025-08-13 14:56:46,768] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False +Starting enhanced contrastive training... +- Contrastive learning enabled +- Protein-text matching: enabled +- Temperature: 0.07 +- PTM weight: 1.0 + 0%| | 0/67290 [00:00 + trainer = main(args) + ^^^^^^^^^^ + File "/nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py", line 399, in main + trainer.train() + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/trainer.py", line 2238, in train + return inner_training_loop( + ^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/trainer.py", line 2582, in _inner_training_loop + tr_loss_step = self.training_step(model, inputs, num_items_in_batch) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/trainer.py", line 3796, in training_step + loss = self.compute_loss(model, inputs, num_items_in_batch=num_items_in_batch) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/nas/shared/kilab/wangyujia/BioReason_new/bioreason/trainer/contrast_trainer_new.py", line 303, in compute_loss + sim_p2t, sim_t2p, loss_contrastive = self.contrastive_loss.contrast_global( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/nas/shared/kilab/wangyujia/BioReason_new/bioreason/trainer/contrast_trainer_new.py", line 114, in contrast_global + sim_t2p = (features_text.unsqueeze(1).unsqueeze(1) @ features_protein_all.permute(0, 2, 1)).squeeze() + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +RuntimeError: permute(sparse_coo): number of dimensions in the tensor input does not match the length of the desired ordering of dimensions i.e. input.dim() = 2 is not equal to len(dims) = 3 diff --git a/BioReason_new/wandb/run-20250813_145627-ep0ib6n2/files/requirements.txt b/BioReason_new/wandb/run-20250813_145627-ep0ib6n2/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..743d03fe27e0b7e200935023c07a1d7fa61b99ce --- /dev/null +++ b/BioReason_new/wandb/run-20250813_145627-ep0ib6n2/files/requirements.txt @@ -0,0 +1,233 @@ +jupyterlab_pygments==0.3.0 +depyf==0.19.0 +av==15.0.0 +fastapi==0.116.1 +ray==2.48.0 +cachetools==6.1.0 +prometheus_client==0.22.1 +requests==2.32.4 +jupyter-lsp==2.2.6 +jupyter-events==0.12.0 +anyio==4.10.0 +pybase64==1.4.2 +fastjsonschema==2.21.1 +cloudpickle==3.1.1 +sentry-sdk==2.34.1 +torchmetrics==1.8.1 +wheel==0.45.1 +idna==3.10 +defusedxml==0.7.1 +starlette==0.47.2 +python-dotenv==1.1.1 +comm==0.2.3 +multidict==6.6.3 +python-json-logger==3.3.0 +jupyter_server==2.16.0 +nvidia-nvtx-cu12==12.6.77 +websockets==15.0.1 +referencing==0.36.2 +gguf==0.17.1 +lm-format-enforcer==0.10.12 +xformers==0.0.31 +ninja==1.11.1.4 +beautifulsoup4==4.13.4 +dnspython==2.7.0 +certifi==2025.8.3 +markdown-it-py==3.0.0 +stack-data==0.6.3 +nvidia-cusparselt-cu12==0.6.3 +networkx==3.5 +aiohappyeyeballs==2.6.1 +nvidia-cuda-cupti-cu12==12.6.80 +llguidance==0.7.30 +rich==14.1.0 +outlines_core==0.2.10 +nvidia-nccl-cu12==2.26.2 +pandas==2.3.1 +pydantic_core==2.33.2 +argon2-cffi==25.1.0 +babel==2.17.0 +isoduration==20.11.0 +uvloop==0.21.0 +json5==0.12.0 +widgetsnbextension==4.0.14 +aiohttp==3.12.15 +notebook_shim==0.2.4 +pandocfilters==1.5.1 +vllm==0.10.0 +blake3==1.0.5 +diskcache==5.6.3 +h11==0.16.0 +attrs==25.3.0 +nvidia-cusolver-cu12==11.7.1.2 +click==8.2.1 +cupy-cuda12x==13.5.1 +jedi==0.19.2 +py-cpuinfo==9.0.0 +overrides==7.7.0 +pytz==2025.2 +protobuf==6.31.1 +dill==0.3.8 +ipykernel==6.30.1 +frozenlist==1.7.0 +huggingface-hub==0.34.4 +rignore==0.6.4 +mistune==3.1.3 +sniffio==1.3.1 +httpx==0.28.1 +sentencepiece==0.2.0 +typing_extensions==4.14.1 +typing-inspection==0.4.1 +async-lru==2.0.5 +bioreason==0.1.0 +charset-normalizer==3.4.3 +llvmlite==0.44.0 +safetensors==0.6.2 +fqdn==1.5.1 +peft==0.17.0 +fastapi-cli==0.0.8 +torch==2.7.1 +jsonschema==4.25.0 +torchvision==0.22.1 +parso==0.8.4 +pillow==11.3.0 +pycountry==24.6.1 +scipy==1.16.1 +wandb==0.21.1 +pip==25.1 +wcwidth==0.2.13 +mistral_common==1.8.3 +nvidia-cudnn-cu12==9.5.1.17 +asttokens==3.0.0 +numpy==2.2.6 +uvicorn==0.35.0 +jupyter_client==8.6.3 +debugpy==1.8.16 +fastapi-cloud-cli==0.1.5 +compressed-tensors==0.10.2 +GitPython==3.1.45 +nvidia-ml-py==13.580.65 +tiktoken==0.11.0 +partial-json-parser==0.2.1.1.post6 +prompt_toolkit==3.0.51 +uri-template==1.3.0 +pyarrow==21.0.0 +nvidia-cuda-nvrtc-cu12==12.6.77 +propcache==0.3.2 +matplotlib-inline==0.1.7 +pure_eval==0.2.3 +nest-asyncio==1.6.0 +deepspeed==0.17.4 +einops==0.8.1 +triton==3.3.1 +trl==0.21.0 +pyzmq==27.0.1 +httptools==0.6.4 +ipython==9.4.0 +packaging==25.0 +hjson==3.1.0 +soxr==0.5.0.post1 +webcolors==24.11.1 +PyYAML==6.0.2 +ipywidgets==8.1.7 +lark==1.2.2 +python-multipart==0.0.20 +tinycss2==1.4.0 +mpmath==1.3.0 +psutil==7.0.0 +nbformat==5.10.4 +qwen-vl-utils==0.0.11 +notebook==7.4.5 +jupyterlab==4.4.5 +transformers==4.55.0 +urllib3==2.5.0 +terminado==0.18.1 +xgrammar==0.1.21 +xxhash==3.5.0 +tzdata==2025.2 +jupyterlab_widgets==3.0.15 +aiosignal==1.4.0 +annotated-types==0.7.0 +nvidia-nvjitlink-cu12==12.6.85 +httpcore==1.0.9 +pexpect==4.9.0 +tornado==6.5.2 +regex==2025.7.33 +filelock==3.18.0 +nvidia-cuda-runtime-cu12==12.6.77 +openai==1.90.0 +Pygments==2.19.2 +cffi==1.17.1 +nvidia-cufft-cu12==11.3.0.4 +rpds-py==0.27.0 +rich-toolkit==0.15.0 +bleach==6.2.0 +sympy==1.14.0 +python-dateutil==2.9.0.post0 +watchfiles==1.1.0 +nbclient==0.10.2 +hf-xet==1.1.7 +Send2Trash==1.8.3 +smmap==5.0.2 +tokenizers==0.21.4 +yarl==1.20.1 +MarkupSafe==3.0.2 +prometheus-fastapi-instrumentator==7.1.0 +rfc3986-validator==0.1.1 +bitsandbytes==0.46.1 +nvidia-cufile-cu12==1.11.1.6 +fsspec==2025.3.0 +executing==2.2.0 +pydantic==2.11.7 +jupyter_core==5.8.1 +msgspec==0.19.0 +nvidia-cusparse-cu12==12.5.4.2 +opencv-python-headless==4.12.0.88 +webencodings==0.5.1 +ptyprocess==0.7.0 +pytorch-lightning==2.5.2 +Jinja2==3.1.6 +websocket-client==1.8.0 +jsonpointer==3.0.0 +traitlets==5.14.3 +numba==0.61.2 +soupsieve==2.7 +cbor2==5.6.5 +multiprocess==0.70.16 +email_validator==2.2.0 +distro==1.9.0 +accelerate==1.10.0 +torchaudio==2.7.1 +jiter==0.10.0 +rfc3339-validator==0.1.4 +jupyter==1.1.1 +typer==0.16.0 +gitdb==4.0.12 +types-python-dateutil==2.9.0.20250809 +soundfile==0.13.1 +astor==0.8.1 +shellingham==1.5.4 +jupyterlab_server==2.27.3 +nvidia-curand-cu12==10.3.7.77 +jupyter-console==6.6.3 +msgpack==1.1.1 +six==1.17.0 +pycparser==2.22 +arrow==1.3.0 +tqdm==4.67.1 +lightning-utilities==0.15.2 +interegular==0.3.3 +rfc3987-syntax==1.1.0 +ipython_pygments_lexers==1.1.1 +datasets==4.0.0 +jupyter_server_terminals==0.5.3 +platformdirs==4.3.8 +pydantic-extra-types==2.10.5 +decorator==5.2.1 +mdurl==0.1.2 +nvidia-cublas-cu12==12.6.4.1 +jsonschema-specifications==2025.4.1 +setuptools==78.1.1 +fastrlock==0.8.3 +argon2-cffi-bindings==25.1.0 +nbconvert==7.16.6 diff --git a/BioReason_new/wandb/run-20250813_145627-ep0ib6n2/files/wandb-metadata.json b/BioReason_new/wandb/run-20250813_145627-ep0ib6n2/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..06ca794d5a90962367f108ad2282d9a6cc468061 --- /dev/null +++ b/BioReason_new/wandb/run-20250813_145627-ep0ib6n2/files/wandb-metadata.json @@ -0,0 +1,116 @@ +{ + "os": "Linux-5.10.134-008.18.kangaroo.al8.x86_64-x86_64-with-glibc2.35", + "python": "CPython 3.11.0", + "startedAt": "2025-08-13T06:56:27.037635Z", + "args": [ + "--text_model_name", + "/oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged", + "--protein_model_name", + "/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m", + "--qformer_model_name", + "/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft", + "--num_query_tokens", + "8", + "--train_dataset", + "/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl", + "--valid_dataset", + "/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl", + "--output_dir", + "./contrastive_outputs", + "--num_epochs", + "10", + "--batch_size", + "8", + "--learning_rate", + "1e-4", + "--temperature", + "0.07", + "--freeze_protein_model", + "--freeze_text_model", + "--enable_ptm", + "--max_length_protein", + "1024", + "--max_length_text", + "512", + "--num_workers", + "8", + "--eval_dataset", + "--use_wandb", + "--wandb_project", + "protein-llm-contrastive", + "--logging_steps", + "100", + "--eval_steps", + "500", + "--save_steps", + "1000" + ], + "program": "/nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py", + "codePath": "wangyujia/BioReason_new/train_contrastive.py", + "codePathLocal": "train_contrastive.py", + "git": { + "remote": "https://github.com/PorUna-byte/PAR.git", + "commit": "b8caf406aa1699c788f0ca6e44a1769452c317db" + }, + "email": "gia0603yucca@gmail.com", + "root": "/nas/shared/kilab/wangyujia/BioReason_new", + "host": "dsw-265304-585cc9d768-ckfd7", + "executable": "/root/miniconda3/envs/bioreason/bin/python", + "cpu_count": 64, + "cpu_count_logical": 64, + "gpu": "NVIDIA A800-SXM4-80GB", + "gpu_count": 8, + "disk": { + "/": { + "total": "1623302262784", + "used": "1015189504" + } + }, + "memory": { + "total": "549755813888" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-62fcc532-880f-7324-eb13-3b739a38c29b" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-af1197aa-3e7d-a53e-185d-04b1d64164da" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-3e6e551f-2d7d-a27b-654a-da56933d2283" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-a947b64a-c545-ee5e-30e6-b4b0e2dd8fdc" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-ebd04f3f-c437-1b8b-26f2-4b1ebfee93c4" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-53994891-72a6-25be-a4c8-e32c695155b0" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-a927d00a-15c5-31f5-9ae1-9d2e773821a1" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-93d555c4-29a0-cd78-abac-d98cd160d4e1" + } + ], + "cudaVersion": "12.1", + "writerId": "8kepep2ysih7kn77h1iftipfg3shqy5q" +} \ No newline at end of file diff --git a/BioReason_new/wandb/run-20250813_145627-ep0ib6n2/files/wandb-summary.json b/BioReason_new/wandb/run-20250813_145627-ep0ib6n2/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..bf2f3842f846b6e4f8b438e77cc5210e1b865227 --- /dev/null +++ b/BioReason_new/wandb/run-20250813_145627-ep0ib6n2/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":18},"_runtime":18} \ No newline at end of file diff --git a/BioReason_new/wandb/run-20250813_145627-ep0ib6n2/logs/debug-internal.log b/BioReason_new/wandb/run-20250813_145627-ep0ib6n2/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..82a2c05ba868885837b9f8ab48b4647c1d552f59 --- /dev/null +++ b/BioReason_new/wandb/run-20250813_145627-ep0ib6n2/logs/debug-internal.log @@ -0,0 +1,39 @@ +{"time":"2025-08-13T14:56:27.316763198+08:00","level":"INFO","msg":"stream: starting","core version":"0.21.1"} +{"time":"2025-08-13T14:56:30.675423119+08:00","level":"INFO","msg":"stream: created new stream","id":"ep0ib6n2"} +{"time":"2025-08-13T14:56:30.676242172+08:00","level":"INFO","msg":"stream: started","id":"ep0ib6n2"} +{"time":"2025-08-13T14:56:30.676275983+08:00","level":"INFO","msg":"sender: started","stream_id":"ep0ib6n2"} +{"time":"2025-08-13T14:56:30.676265881+08:00","level":"INFO","msg":"writer: started","stream_id":"ep0ib6n2"} +{"time":"2025-08-13T14:56:30.676313509+08:00","level":"INFO","msg":"handler: started","stream_id":"ep0ib6n2"} +{"time":"2025-08-13T14:56:35.648007444+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=8a95163e0c0f889892ef15b4a88e43d530288eda1d8571066f8e5a5a448332ea60912856e41297390d209d6ef04c28c92498205a84c3ed6ba3f952108c0e9652e9d14ab6e33add6388a6f39baf85d21db8f98a78b2d267a131a011a5d79b267ba86938c2efe0d867337f5cf8a27924e4ca34fda8e2302d0ddbfa5d507e64601856113db1b9ace6fb3ee615f8f3b05ce7a2fe03f38fd8953c9ed405d6a5ff00bd2c59023b984f8a19d6b9653b17a073d109be52afdedc26e456311889d732a252a242a9388117272376c4cec78ffcef9c8f238bea6b854f7189d3e1e721e175292f5da73859ff0d5f0ac7ee2ca45cad4bb3ea441cfb6d3d91f42f9bf287312182&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:40004->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=8a95163e0c0f889892ef15b4a88e43d530288eda1d8571066f8e5a5a448332ea60912856e41297390d209d6ef04c28c92498205a84c3ed6ba3f952108c0e9652e9d14ab6e33add6388a6f39baf85d21db8f98a78b2d267a131a011a5d79b267ba86938c2efe0d867337f5cf8a27924e4ca34fda8e2302d0ddbfa5d507e64601856113db1b9ace6fb3ee615f8f3b05ce7a2fe03f38fd8953c9ed405d6a5ff00bd2c59023b984f8a19d6b9653b17a073d109be52afdedc26e456311889d732a252a242a9388117272376c4cec78ffcef9c8f238bea6b854f7189d3e1e721e175292f5da73859ff0d5f0ac7ee2ca45cad4bb3ea441cfb6d3d91f42f9bf287312182&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:56:38.348315908+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=8a95163e0c0f889892ef15b4a88e43d530288eda1d8571066f8e5a5a448332ea60912856e41297390d209d6ef04c28c92498205a84c3ed6ba3f952108c0e9652e9d14ab6e33add6388a6f39baf85d21db8f98a78b2d267a131a011a5d79b267ba86938c2efe0d867337f5cf8a27924e4ca34fda8e2302d0ddbfa5d507e64601856113db1b9ace6fb3ee615f8f3b05ce7a2fe03f38fd8953c9ed405d6a5ff00bd2c59023b984f8a19d6b9653b17a073d109be52afdedc26e456311889d732a252a242a9388117272376c4cec78ffcef9c8f238bea6b854f7189d3e1e721e175292f5da73859ff0d5f0ac7ee2ca45cad4bb3ea441cfb6d3d91f42f9bf287312182&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:46218->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=8a95163e0c0f889892ef15b4a88e43d530288eda1d8571066f8e5a5a448332ea60912856e41297390d209d6ef04c28c92498205a84c3ed6ba3f952108c0e9652e9d14ab6e33add6388a6f39baf85d21db8f98a78b2d267a131a011a5d79b267ba86938c2efe0d867337f5cf8a27924e4ca34fda8e2302d0ddbfa5d507e64601856113db1b9ace6fb3ee615f8f3b05ce7a2fe03f38fd8953c9ed405d6a5ff00bd2c59023b984f8a19d6b9653b17a073d109be52afdedc26e456311889d732a252a242a9388117272376c4cec78ffcef9c8f238bea6b854f7189d3e1e721e175292f5da73859ff0d5f0ac7ee2ca45cad4bb3ea441cfb6d3d91f42f9bf287312182&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:56:43.61700136+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=8a95163e0c0f889892ef15b4a88e43d530288eda1d8571066f8e5a5a448332ea60912856e41297390d209d6ef04c28c92498205a84c3ed6ba3f952108c0e9652e9d14ab6e33add6388a6f39baf85d21db8f98a78b2d267a131a011a5d79b267ba86938c2efe0d867337f5cf8a27924e4ca34fda8e2302d0ddbfa5d507e64601856113db1b9ace6fb3ee615f8f3b05ce7a2fe03f38fd8953c9ed405d6a5ff00bd2c59023b984f8a19d6b9653b17a073d109be52afdedc26e456311889d732a252a242a9388117272376c4cec78ffcef9c8f238bea6b854f7189d3e1e721e175292f5da73859ff0d5f0ac7ee2ca45cad4bb3ea441cfb6d3d91f42f9bf287312182&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:46222->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=8a95163e0c0f889892ef15b4a88e43d530288eda1d8571066f8e5a5a448332ea60912856e41297390d209d6ef04c28c92498205a84c3ed6ba3f952108c0e9652e9d14ab6e33add6388a6f39baf85d21db8f98a78b2d267a131a011a5d79b267ba86938c2efe0d867337f5cf8a27924e4ca34fda8e2302d0ddbfa5d507e64601856113db1b9ace6fb3ee615f8f3b05ce7a2fe03f38fd8953c9ed405d6a5ff00bd2c59023b984f8a19d6b9653b17a073d109be52afdedc26e456311889d732a252a242a9388117272376c4cec78ffcef9c8f238bea6b854f7189d3e1e721e175292f5da73859ff0d5f0ac7ee2ca45cad4bb3ea441cfb6d3d91f42f9bf287312182&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:56:51.69459817+08:00","level":"INFO","msg":"stream: closing","id":"ep0ib6n2"} +{"time":"2025-08-13T14:56:52.777037066+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=8a95163e0c0f889892ef15b4a88e43d530288eda1d8571066f8e5a5a448332ea60912856e41297390d209d6ef04c28c92498205a84c3ed6ba3f952108c0e9652e9d14ab6e33add6388a6f39baf85d21db8f98a78b2d267a131a011a5d79b267ba86938c2efe0d867337f5cf8a27924e4ca34fda8e2302d0ddbfa5d507e64601856113db1b9ace6fb3ee615f8f3b05ce7a2fe03f38fd8953c9ed405d6a5ff00bd2c59023b984f8a19d6b9653b17a073d109be52afdedc26e456311889d732a252a242a9388117272376c4cec78ffcef9c8f238bea6b854f7189d3e1e721e175292f5da73859ff0d5f0ac7ee2ca45cad4bb3ea441cfb6d3d91f42f9bf287312182&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:36500->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=8a95163e0c0f889892ef15b4a88e43d530288eda1d8571066f8e5a5a448332ea60912856e41297390d209d6ef04c28c92498205a84c3ed6ba3f952108c0e9652e9d14ab6e33add6388a6f39baf85d21db8f98a78b2d267a131a011a5d79b267ba86938c2efe0d867337f5cf8a27924e4ca34fda8e2302d0ddbfa5d507e64601856113db1b9ace6fb3ee615f8f3b05ce7a2fe03f38fd8953c9ed405d6a5ff00bd2c59023b984f8a19d6b9653b17a073d109be52afdedc26e456311889d732a252a242a9388117272376c4cec78ffcef9c8f238bea6b854f7189d3e1e721e175292f5da73859ff0d5f0ac7ee2ca45cad4bb3ea441cfb6d3d91f42f9bf287312182&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:58:45.417748957+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=92ae95a068573a7caa07db2b3bcab253490e0ebe38e0fa761f5a93e49b4c4a5d8e2e667590c2c9e17deb189bbaefcf1b894a2011e61c1124d05ae9a8e9bec1f9d63d6def9d54a24b98f45661ad1a17719ea840a41176dc66ac4ed14242c2d74e4ba8600bdddc171e2f3665abd1a0e289176017c0265fe3c49df53270fd495056a1154d7151533eddfd23de83ffc0ec604a59a24e68e2124c9461b71269bfd51f971767d3402294f3a605f7d2c69730216b6b9e3f14a94e38918150fa9ff89bed4ca0449595c0b2392cbd9fd87b07910c18ceaaf32de694b69ee7d3c9845d3c6ff422849de5ae0bc2018170c0533c41df905d8af0f66ea3ba4713d1810814ca00&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:55988->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=92ae95a068573a7caa07db2b3bcab253490e0ebe38e0fa761f5a93e49b4c4a5d8e2e667590c2c9e17deb189bbaefcf1b894a2011e61c1124d05ae9a8e9bec1f9d63d6def9d54a24b98f45661ad1a17719ea840a41176dc66ac4ed14242c2d74e4ba8600bdddc171e2f3665abd1a0e289176017c0265fe3c49df53270fd495056a1154d7151533eddfd23de83ffc0ec604a59a24e68e2124c9461b71269bfd51f971767d3402294f3a605f7d2c69730216b6b9e3f14a94e38918150fa9ff89bed4ca0449595c0b2392cbd9fd87b07910c18ceaaf32de694b69ee7d3c9845d3c6ff422849de5ae0bc2018170c0533c41df905d8af0f66ea3ba4713d1810814ca00&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:59:10.024483291+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065657Z&X-Goog-Expires=86399&X-Goog-Signature=b499547cdb15cac87e48fd391f9ed5dad4148a5e06655bd6e6ad06f7feb1fe9ec68cf9977d5655d6f0d89419803ffdf41be6376849cf43a02f5131adf34a050997e1d8d7cb7ab1fd893c7419d00769e0721e52a462ae922282e98feb03c1a136b4f3432c28f72e3a7f0a08efbcd101fb4dfa14e0959db1d1908cebae7ee00098b30a81916a127012f6418d1a91a93cfdc2f330bd80726446f2715bfe93a3117b6c1792f2067565ad2ab643e19967291f785e76d1c2a8d25402682089fc241b6fd8dbc721d8e51fb830fa0a6a83aabe60a39eaf11c1ffd475a406852af5563e870416b290e5e3e50c867afb12812a5a1433d3630064613b00ac1e56e148b48cd5&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:38146->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065657Z&X-Goog-Expires=86399&X-Goog-Signature=b499547cdb15cac87e48fd391f9ed5dad4148a5e06655bd6e6ad06f7feb1fe9ec68cf9977d5655d6f0d89419803ffdf41be6376849cf43a02f5131adf34a050997e1d8d7cb7ab1fd893c7419d00769e0721e52a462ae922282e98feb03c1a136b4f3432c28f72e3a7f0a08efbcd101fb4dfa14e0959db1d1908cebae7ee00098b30a81916a127012f6418d1a91a93cfdc2f330bd80726446f2715bfe93a3117b6c1792f2067565ad2ab643e19967291f785e76d1c2a8d25402682089fc241b6fd8dbc721d8e51fb830fa0a6a83aabe60a39eaf11c1ffd475a406852af5563e870416b290e5e3e50c867afb12812a5a1433d3630064613b00ac1e56e148b48cd5&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T14:59:12.58298548+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065657Z&X-Goog-Expires=86399&X-Goog-Signature=b499547cdb15cac87e48fd391f9ed5dad4148a5e06655bd6e6ad06f7feb1fe9ec68cf9977d5655d6f0d89419803ffdf41be6376849cf43a02f5131adf34a050997e1d8d7cb7ab1fd893c7419d00769e0721e52a462ae922282e98feb03c1a136b4f3432c28f72e3a7f0a08efbcd101fb4dfa14e0959db1d1908cebae7ee00098b30a81916a127012f6418d1a91a93cfdc2f330bd80726446f2715bfe93a3117b6c1792f2067565ad2ab643e19967291f785e76d1c2a8d25402682089fc241b6fd8dbc721d8e51fb830fa0a6a83aabe60a39eaf11c1ffd475a406852af5563e870416b290e5e3e50c867afb12812a5a1433d3630064613b00ac1e56e148b48cd5&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:38162->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065657Z&X-Goog-Expires=86399&X-Goog-Signature=b499547cdb15cac87e48fd391f9ed5dad4148a5e06655bd6e6ad06f7feb1fe9ec68cf9977d5655d6f0d89419803ffdf41be6376849cf43a02f5131adf34a050997e1d8d7cb7ab1fd893c7419d00769e0721e52a462ae922282e98feb03c1a136b4f3432c28f72e3a7f0a08efbcd101fb4dfa14e0959db1d1908cebae7ee00098b30a81916a127012f6418d1a91a93cfdc2f330bd80726446f2715bfe93a3117b6c1792f2067565ad2ab643e19967291f785e76d1c2a8d25402682089fc241b6fd8dbc721d8e51fb830fa0a6a83aabe60a39eaf11c1ffd475a406852af5563e870416b290e5e3e50c867afb12812a5a1433d3630064613b00ac1e56e148b48cd5&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:00:56.440152443+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=92ae95a068573a7caa07db2b3bcab253490e0ebe38e0fa761f5a93e49b4c4a5d8e2e667590c2c9e17deb189bbaefcf1b894a2011e61c1124d05ae9a8e9bec1f9d63d6def9d54a24b98f45661ad1a17719ea840a41176dc66ac4ed14242c2d74e4ba8600bdddc171e2f3665abd1a0e289176017c0265fe3c49df53270fd495056a1154d7151533eddfd23de83ffc0ec604a59a24e68e2124c9461b71269bfd51f971767d3402294f3a605f7d2c69730216b6b9e3f14a94e38918150fa9ff89bed4ca0449595c0b2392cbd9fd87b07910c18ceaaf32de694b69ee7d3c9845d3c6ff422849de5ae0bc2018170c0533c41df905d8af0f66ea3ba4713d1810814ca00&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:59240->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=92ae95a068573a7caa07db2b3bcab253490e0ebe38e0fa761f5a93e49b4c4a5d8e2e667590c2c9e17deb189bbaefcf1b894a2011e61c1124d05ae9a8e9bec1f9d63d6def9d54a24b98f45661ad1a17719ea840a41176dc66ac4ed14242c2d74e4ba8600bdddc171e2f3665abd1a0e289176017c0265fe3c49df53270fd495056a1154d7151533eddfd23de83ffc0ec604a59a24e68e2124c9461b71269bfd51f971767d3402294f3a605f7d2c69730216b6b9e3f14a94e38918150fa9ff89bed4ca0449595c0b2392cbd9fd87b07910c18ceaaf32de694b69ee7d3c9845d3c6ff422849de5ae0bc2018170c0533c41df905d8af0f66ea3ba4713d1810814ca00&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:01:21.034779714+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065652Z&X-Goog-Expires=86399&X-Goog-Signature=70e86fac9315b3f1b972a2e965d19160dd68224c29b64e26d1273b1cf0ee0cd71a46b173414db97392275bb488e58c14002348a4e91b03b1a747512a2ab3b841817edc1288805b4177ab9838fb3417705a17bd647a0d9441f2a9d0f1788c3cb687ab8169e5be11b49506785f6ed913ec58a1cabca533c781c70847d44493af27a7f03875ceb771ebef489a0b4e4b5fa84c501f8a784c8f4a0113ea74af2ef4ea840a2a052e33a1f6b3e354761f6966301b00de787bf35a8f259f43f5c6b298ba7ce56f19efec23286c40af589f8d136ba02a4e8c6ed0ec70ae825ed54fcd70fa5acdd75ae8a4d17a34a554fa3b6f3a2482d9462df72ae4cca800cf50f008bb6a&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:53078->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065652Z&X-Goog-Expires=86399&X-Goog-Signature=70e86fac9315b3f1b972a2e965d19160dd68224c29b64e26d1273b1cf0ee0cd71a46b173414db97392275bb488e58c14002348a4e91b03b1a747512a2ab3b841817edc1288805b4177ab9838fb3417705a17bd647a0d9441f2a9d0f1788c3cb687ab8169e5be11b49506785f6ed913ec58a1cabca533c781c70847d44493af27a7f03875ceb771ebef489a0b4e4b5fa84c501f8a784c8f4a0113ea74af2ef4ea840a2a052e33a1f6b3e354761f6966301b00de787bf35a8f259f43f5c6b298ba7ce56f19efec23286c40af589f8d136ba02a4e8c6ed0ec70ae825ed54fcd70fa5acdd75ae8a4d17a34a554fa3b6f3a2482d9462df72ae4cca800cf50f008bb6a&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:01:21.04577582+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065652Z&X-Goog-Expires=86399&X-Goog-Signature=90ed12f3fe9c9b10f930fadccdb3bf6688307cfcb66aa11a69d0f96047f0ff09401ed55e9eb0cb66cbd9d383845cd893c0156b96247b225f27a027b4dc63458a22e352ddf0db6cfd76d6737e9a3197d7ed8937a9550a5b8a9256d656be8b242e54fde209064a20f89273fa6fcaa8086fd9c43159c4782be5263a45e453d93b8c02e34e41a6371ac8052670756a97db99e0c5e679fd89f8005910eeed307e69a82bdc2998e602effba1d7522c5d65ac371bd161f447c40092e29f017f7bfd56c41ecb3eff9690ce74b62251fecf37078861335e37aa8d57bd4c2325dbc55a727fbc627b2e68156e3e8e7c70da5b4ef4c0032dfaa57f327bc58f069084d184a500&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:53070->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065652Z&X-Goog-Expires=86399&X-Goog-Signature=90ed12f3fe9c9b10f930fadccdb3bf6688307cfcb66aa11a69d0f96047f0ff09401ed55e9eb0cb66cbd9d383845cd893c0156b96247b225f27a027b4dc63458a22e352ddf0db6cfd76d6737e9a3197d7ed8937a9550a5b8a9256d656be8b242e54fde209064a20f89273fa6fcaa8086fd9c43159c4782be5263a45e453d93b8c02e34e41a6371ac8052670756a97db99e0c5e679fd89f8005910eeed307e69a82bdc2998e602effba1d7522c5d65ac371bd161f447c40092e29f017f7bfd56c41ecb3eff9690ce74b62251fecf37078861335e37aa8d57bd4c2325dbc55a727fbc627b2e68156e3e8e7c70da5b4ef4c0032dfaa57f327bc58f069084d184a500&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:01:33.329184944+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=8a95163e0c0f889892ef15b4a88e43d530288eda1d8571066f8e5a5a448332ea60912856e41297390d209d6ef04c28c92498205a84c3ed6ba3f952108c0e9652e9d14ab6e33add6388a6f39baf85d21db8f98a78b2d267a131a011a5d79b267ba86938c2efe0d867337f5cf8a27924e4ca34fda8e2302d0ddbfa5d507e64601856113db1b9ace6fb3ee615f8f3b05ce7a2fe03f38fd8953c9ed405d6a5ff00bd2c59023b984f8a19d6b9653b17a073d109be52afdedc26e456311889d732a252a242a9388117272376c4cec78ffcef9c8f238bea6b854f7189d3e1e721e175292f5da73859ff0d5f0ac7ee2ca45cad4bb3ea441cfb6d3d91f42f9bf287312182&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:54832->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=8a95163e0c0f889892ef15b4a88e43d530288eda1d8571066f8e5a5a448332ea60912856e41297390d209d6ef04c28c92498205a84c3ed6ba3f952108c0e9652e9d14ab6e33add6388a6f39baf85d21db8f98a78b2d267a131a011a5d79b267ba86938c2efe0d867337f5cf8a27924e4ca34fda8e2302d0ddbfa5d507e64601856113db1b9ace6fb3ee615f8f3b05ce7a2fe03f38fd8953c9ed405d6a5ff00bd2c59023b984f8a19d6b9653b17a073d109be52afdedc26e456311889d732a252a242a9388117272376c4cec78ffcef9c8f238bea6b854f7189d3e1e721e175292f5da73859ff0d5f0ac7ee2ca45cad4bb3ea441cfb6d3d91f42f9bf287312182&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:03:11.684216561+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=92ae95a068573a7caa07db2b3bcab253490e0ebe38e0fa761f5a93e49b4c4a5d8e2e667590c2c9e17deb189bbaefcf1b894a2011e61c1124d05ae9a8e9bec1f9d63d6def9d54a24b98f45661ad1a17719ea840a41176dc66ac4ed14242c2d74e4ba8600bdddc171e2f3665abd1a0e289176017c0265fe3c49df53270fd495056a1154d7151533eddfd23de83ffc0ec604a59a24e68e2124c9461b71269bfd51f971767d3402294f3a605f7d2c69730216b6b9e3f14a94e38918150fa9ff89bed4ca0449595c0b2392cbd9fd87b07910c18ceaaf32de694b69ee7d3c9845d3c6ff422849de5ae0bc2018170c0533c41df905d8af0f66ea3ba4713d1810814ca00&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:44832->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=92ae95a068573a7caa07db2b3bcab253490e0ebe38e0fa761f5a93e49b4c4a5d8e2e667590c2c9e17deb189bbaefcf1b894a2011e61c1124d05ae9a8e9bec1f9d63d6def9d54a24b98f45661ad1a17719ea840a41176dc66ac4ed14242c2d74e4ba8600bdddc171e2f3665abd1a0e289176017c0265fe3c49df53270fd495056a1154d7151533eddfd23de83ffc0ec604a59a24e68e2124c9461b71269bfd51f971767d3402294f3a605f7d2c69730216b6b9e3f14a94e38918150fa9ff89bed4ca0449595c0b2392cbd9fd87b07910c18ceaaf32de694b69ee7d3c9845d3c6ff422849de5ae0bc2018170c0533c41df905d8af0f66ea3ba4713d1810814ca00&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:03:21.627651503+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=92ae95a068573a7caa07db2b3bcab253490e0ebe38e0fa761f5a93e49b4c4a5d8e2e667590c2c9e17deb189bbaefcf1b894a2011e61c1124d05ae9a8e9bec1f9d63d6def9d54a24b98f45661ad1a17719ea840a41176dc66ac4ed14242c2d74e4ba8600bdddc171e2f3665abd1a0e289176017c0265fe3c49df53270fd495056a1154d7151533eddfd23de83ffc0ec604a59a24e68e2124c9461b71269bfd51f971767d3402294f3a605f7d2c69730216b6b9e3f14a94e38918150fa9ff89bed4ca0449595c0b2392cbd9fd87b07910c18ceaaf32de694b69ee7d3c9845d3c6ff422849de5ae0bc2018170c0533c41df905d8af0f66ea3ba4713d1810814ca00&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:45918->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=92ae95a068573a7caa07db2b3bcab253490e0ebe38e0fa761f5a93e49b4c4a5d8e2e667590c2c9e17deb189bbaefcf1b894a2011e61c1124d05ae9a8e9bec1f9d63d6def9d54a24b98f45661ad1a17719ea840a41176dc66ac4ed14242c2d74e4ba8600bdddc171e2f3665abd1a0e289176017c0265fe3c49df53270fd495056a1154d7151533eddfd23de83ffc0ec604a59a24e68e2124c9461b71269bfd51f971767d3402294f3a605f7d2c69730216b6b9e3f14a94e38918150fa9ff89bed4ca0449595c0b2392cbd9fd87b07910c18ceaaf32de694b69ee7d3c9845d3c6ff422849de5ae0bc2018170c0533c41df905d8af0f66ea3ba4713d1810814ca00&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:03:32.12823412+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065652Z&X-Goog-Expires=86399&X-Goog-Signature=70e86fac9315b3f1b972a2e965d19160dd68224c29b64e26d1273b1cf0ee0cd71a46b173414db97392275bb488e58c14002348a4e91b03b1a747512a2ab3b841817edc1288805b4177ab9838fb3417705a17bd647a0d9441f2a9d0f1788c3cb687ab8169e5be11b49506785f6ed913ec58a1cabca533c781c70847d44493af27a7f03875ceb771ebef489a0b4e4b5fa84c501f8a784c8f4a0113ea74af2ef4ea840a2a052e33a1f6b3e354761f6966301b00de787bf35a8f259f43f5c6b298ba7ce56f19efec23286c40af589f8d136ba02a4e8c6ed0ec70ae825ed54fcd70fa5acdd75ae8a4d17a34a554fa3b6f3a2482d9462df72ae4cca800cf50f008bb6a&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:48788->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065652Z&X-Goog-Expires=86399&X-Goog-Signature=70e86fac9315b3f1b972a2e965d19160dd68224c29b64e26d1273b1cf0ee0cd71a46b173414db97392275bb488e58c14002348a4e91b03b1a747512a2ab3b841817edc1288805b4177ab9838fb3417705a17bd647a0d9441f2a9d0f1788c3cb687ab8169e5be11b49506785f6ed913ec58a1cabca533c781c70847d44493af27a7f03875ceb771ebef489a0b4e4b5fa84c501f8a784c8f4a0113ea74af2ef4ea840a2a052e33a1f6b3e354761f6966301b00de787bf35a8f259f43f5c6b298ba7ce56f19efec23286c40af589f8d136ba02a4e8c6ed0ec70ae825ed54fcd70fa5acdd75ae8a4d17a34a554fa3b6f3a2482d9462df72ae4cca800cf50f008bb6a&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:03:32.147257574+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065652Z&X-Goog-Expires=86399&X-Goog-Signature=90ed12f3fe9c9b10f930fadccdb3bf6688307cfcb66aa11a69d0f96047f0ff09401ed55e9eb0cb66cbd9d383845cd893c0156b96247b225f27a027b4dc63458a22e352ddf0db6cfd76d6737e9a3197d7ed8937a9550a5b8a9256d656be8b242e54fde209064a20f89273fa6fcaa8086fd9c43159c4782be5263a45e453d93b8c02e34e41a6371ac8052670756a97db99e0c5e679fd89f8005910eeed307e69a82bdc2998e602effba1d7522c5d65ac371bd161f447c40092e29f017f7bfd56c41ecb3eff9690ce74b62251fecf37078861335e37aa8d57bd4c2325dbc55a727fbc627b2e68156e3e8e7c70da5b4ef4c0032dfaa57f327bc58f069084d184a500&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:48796->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065652Z&X-Goog-Expires=86399&X-Goog-Signature=90ed12f3fe9c9b10f930fadccdb3bf6688307cfcb66aa11a69d0f96047f0ff09401ed55e9eb0cb66cbd9d383845cd893c0156b96247b225f27a027b4dc63458a22e352ddf0db6cfd76d6737e9a3197d7ed8937a9550a5b8a9256d656be8b242e54fde209064a20f89273fa6fcaa8086fd9c43159c4782be5263a45e453d93b8c02e34e41a6371ac8052670756a97db99e0c5e679fd89f8005910eeed307e69a82bdc2998e602effba1d7522c5d65ac371bd161f447c40092e29f017f7bfd56c41ecb3eff9690ce74b62251fecf37078861335e37aa8d57bd4c2325dbc55a727fbc627b2e68156e3e8e7c70da5b4ef4c0032dfaa57f327bc58f069084d184a500&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:03:38.988365667+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=92ae95a068573a7caa07db2b3bcab253490e0ebe38e0fa761f5a93e49b4c4a5d8e2e667590c2c9e17deb189bbaefcf1b894a2011e61c1124d05ae9a8e9bec1f9d63d6def9d54a24b98f45661ad1a17719ea840a41176dc66ac4ed14242c2d74e4ba8600bdddc171e2f3665abd1a0e289176017c0265fe3c49df53270fd495056a1154d7151533eddfd23de83ffc0ec604a59a24e68e2124c9461b71269bfd51f971767d3402294f3a605f7d2c69730216b6b9e3f14a94e38918150fa9ff89bed4ca0449595c0b2392cbd9fd87b07910c18ceaaf32de694b69ee7d3c9845d3c6ff422849de5ae0bc2018170c0533c41df905d8af0f66ea3ba4713d1810814ca00&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:60578->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=92ae95a068573a7caa07db2b3bcab253490e0ebe38e0fa761f5a93e49b4c4a5d8e2e667590c2c9e17deb189bbaefcf1b894a2011e61c1124d05ae9a8e9bec1f9d63d6def9d54a24b98f45661ad1a17719ea840a41176dc66ac4ed14242c2d74e4ba8600bdddc171e2f3665abd1a0e289176017c0265fe3c49df53270fd495056a1154d7151533eddfd23de83ffc0ec604a59a24e68e2124c9461b71269bfd51f971767d3402294f3a605f7d2c69730216b6b9e3f14a94e38918150fa9ff89bed4ca0449595c0b2392cbd9fd87b07910c18ceaaf32de694b69ee7d3c9845d3c6ff422849de5ae0bc2018170c0533c41df905d8af0f66ea3ba4713d1810814ca00&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:03:40.306186375+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065657Z&X-Goog-Expires=86399&X-Goog-Signature=b499547cdb15cac87e48fd391f9ed5dad4148a5e06655bd6e6ad06f7feb1fe9ec68cf9977d5655d6f0d89419803ffdf41be6376849cf43a02f5131adf34a050997e1d8d7cb7ab1fd893c7419d00769e0721e52a462ae922282e98feb03c1a136b4f3432c28f72e3a7f0a08efbcd101fb4dfa14e0959db1d1908cebae7ee00098b30a81916a127012f6418d1a91a93cfdc2f330bd80726446f2715bfe93a3117b6c1792f2067565ad2ab643e19967291f785e76d1c2a8d25402682089fc241b6fd8dbc721d8e51fb830fa0a6a83aabe60a39eaf11c1ffd475a406852af5563e870416b290e5e3e50c867afb12812a5a1433d3630064613b00ac1e56e148b48cd5&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:45266->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065657Z&X-Goog-Expires=86399&X-Goog-Signature=b499547cdb15cac87e48fd391f9ed5dad4148a5e06655bd6e6ad06f7feb1fe9ec68cf9977d5655d6f0d89419803ffdf41be6376849cf43a02f5131adf34a050997e1d8d7cb7ab1fd893c7419d00769e0721e52a462ae922282e98feb03c1a136b4f3432c28f72e3a7f0a08efbcd101fb4dfa14e0959db1d1908cebae7ee00098b30a81916a127012f6418d1a91a93cfdc2f330bd80726446f2715bfe93a3117b6c1792f2067565ad2ab643e19967291f785e76d1c2a8d25402682089fc241b6fd8dbc721d8e51fb830fa0a6a83aabe60a39eaf11c1ffd475a406852af5563e870416b290e5e3e50c867afb12812a5a1433d3630064613b00ac1e56e148b48cd5&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:04:17.028708323+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=92ae95a068573a7caa07db2b3bcab253490e0ebe38e0fa761f5a93e49b4c4a5d8e2e667590c2c9e17deb189bbaefcf1b894a2011e61c1124d05ae9a8e9bec1f9d63d6def9d54a24b98f45661ad1a17719ea840a41176dc66ac4ed14242c2d74e4ba8600bdddc171e2f3665abd1a0e289176017c0265fe3c49df53270fd495056a1154d7151533eddfd23de83ffc0ec604a59a24e68e2124c9461b71269bfd51f971767d3402294f3a605f7d2c69730216b6b9e3f14a94e38918150fa9ff89bed4ca0449595c0b2392cbd9fd87b07910c18ceaaf32de694b69ee7d3c9845d3c6ff422849de5ae0bc2018170c0533c41df905d8af0f66ea3ba4713d1810814ca00&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:57084->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=92ae95a068573a7caa07db2b3bcab253490e0ebe38e0fa761f5a93e49b4c4a5d8e2e667590c2c9e17deb189bbaefcf1b894a2011e61c1124d05ae9a8e9bec1f9d63d6def9d54a24b98f45661ad1a17719ea840a41176dc66ac4ed14242c2d74e4ba8600bdddc171e2f3665abd1a0e289176017c0265fe3c49df53270fd495056a1154d7151533eddfd23de83ffc0ec604a59a24e68e2124c9461b71269bfd51f971767d3402294f3a605f7d2c69730216b6b9e3f14a94e38918150fa9ff89bed4ca0449595c0b2392cbd9fd87b07910c18ceaaf32de694b69ee7d3c9845d3c6ff422849de5ae0bc2018170c0533c41df905d8af0f66ea3ba4713d1810814ca00&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:05:17.273158453+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=92ae95a068573a7caa07db2b3bcab253490e0ebe38e0fa761f5a93e49b4c4a5d8e2e667590c2c9e17deb189bbaefcf1b894a2011e61c1124d05ae9a8e9bec1f9d63d6def9d54a24b98f45661ad1a17719ea840a41176dc66ac4ed14242c2d74e4ba8600bdddc171e2f3665abd1a0e289176017c0265fe3c49df53270fd495056a1154d7151533eddfd23de83ffc0ec604a59a24e68e2124c9461b71269bfd51f971767d3402294f3a605f7d2c69730216b6b9e3f14a94e38918150fa9ff89bed4ca0449595c0b2392cbd9fd87b07910c18ceaaf32de694b69ee7d3c9845d3c6ff422849de5ae0bc2018170c0533c41df905d8af0f66ea3ba4713d1810814ca00&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:43660->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=92ae95a068573a7caa07db2b3bcab253490e0ebe38e0fa761f5a93e49b4c4a5d8e2e667590c2c9e17deb189bbaefcf1b894a2011e61c1124d05ae9a8e9bec1f9d63d6def9d54a24b98f45661ad1a17719ea840a41176dc66ac4ed14242c2d74e4ba8600bdddc171e2f3665abd1a0e289176017c0265fe3c49df53270fd495056a1154d7151533eddfd23de83ffc0ec604a59a24e68e2124c9461b71269bfd51f971767d3402294f3a605f7d2c69730216b6b9e3f14a94e38918150fa9ff89bed4ca0449595c0b2392cbd9fd87b07910c18ceaaf32de694b69ee7d3c9845d3c6ff422849de5ae0bc2018170c0533c41df905d8af0f66ea3ba4713d1810814ca00&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:05:47.363164165+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065652Z&X-Goog-Expires=86399&X-Goog-Signature=70e86fac9315b3f1b972a2e965d19160dd68224c29b64e26d1273b1cf0ee0cd71a46b173414db97392275bb488e58c14002348a4e91b03b1a747512a2ab3b841817edc1288805b4177ab9838fb3417705a17bd647a0d9441f2a9d0f1788c3cb687ab8169e5be11b49506785f6ed913ec58a1cabca533c781c70847d44493af27a7f03875ceb771ebef489a0b4e4b5fa84c501f8a784c8f4a0113ea74af2ef4ea840a2a052e33a1f6b3e354761f6966301b00de787bf35a8f259f43f5c6b298ba7ce56f19efec23286c40af589f8d136ba02a4e8c6ed0ec70ae825ed54fcd70fa5acdd75ae8a4d17a34a554fa3b6f3a2482d9462df72ae4cca800cf50f008bb6a&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:50704->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065652Z&X-Goog-Expires=86399&X-Goog-Signature=70e86fac9315b3f1b972a2e965d19160dd68224c29b64e26d1273b1cf0ee0cd71a46b173414db97392275bb488e58c14002348a4e91b03b1a747512a2ab3b841817edc1288805b4177ab9838fb3417705a17bd647a0d9441f2a9d0f1788c3cb687ab8169e5be11b49506785f6ed913ec58a1cabca533c781c70847d44493af27a7f03875ceb771ebef489a0b4e4b5fa84c501f8a784c8f4a0113ea74af2ef4ea840a2a052e33a1f6b3e354761f6966301b00de787bf35a8f259f43f5c6b298ba7ce56f19efec23286c40af589f8d136ba02a4e8c6ed0ec70ae825ed54fcd70fa5acdd75ae8a4d17a34a554fa3b6f3a2482d9462df72ae4cca800cf50f008bb6a&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:05:57.828717478+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065652Z&X-Goog-Expires=86399&X-Goog-Signature=70e86fac9315b3f1b972a2e965d19160dd68224c29b64e26d1273b1cf0ee0cd71a46b173414db97392275bb488e58c14002348a4e91b03b1a747512a2ab3b841817edc1288805b4177ab9838fb3417705a17bd647a0d9441f2a9d0f1788c3cb687ab8169e5be11b49506785f6ed913ec58a1cabca533c781c70847d44493af27a7f03875ceb771ebef489a0b4e4b5fa84c501f8a784c8f4a0113ea74af2ef4ea840a2a052e33a1f6b3e354761f6966301b00de787bf35a8f259f43f5c6b298ba7ce56f19efec23286c40af589f8d136ba02a4e8c6ed0ec70ae825ed54fcd70fa5acdd75ae8a4d17a34a554fa3b6f3a2482d9462df72ae4cca800cf50f008bb6a&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:40180->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065652Z&X-Goog-Expires=86399&X-Goog-Signature=70e86fac9315b3f1b972a2e965d19160dd68224c29b64e26d1273b1cf0ee0cd71a46b173414db97392275bb488e58c14002348a4e91b03b1a747512a2ab3b841817edc1288805b4177ab9838fb3417705a17bd647a0d9441f2a9d0f1788c3cb687ab8169e5be11b49506785f6ed913ec58a1cabca533c781c70847d44493af27a7f03875ceb771ebef489a0b4e4b5fa84c501f8a784c8f4a0113ea74af2ef4ea840a2a052e33a1f6b3e354761f6966301b00de787bf35a8f259f43f5c6b298ba7ce56f19efec23286c40af589f8d136ba02a4e8c6ed0ec70ae825ed54fcd70fa5acdd75ae8a4d17a34a554fa3b6f3a2482d9462df72ae4cca800cf50f008bb6a&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:06:16.74966428+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065652Z&X-Goog-Expires=86399&X-Goog-Signature=70e86fac9315b3f1b972a2e965d19160dd68224c29b64e26d1273b1cf0ee0cd71a46b173414db97392275bb488e58c14002348a4e91b03b1a747512a2ab3b841817edc1288805b4177ab9838fb3417705a17bd647a0d9441f2a9d0f1788c3cb687ab8169e5be11b49506785f6ed913ec58a1cabca533c781c70847d44493af27a7f03875ceb771ebef489a0b4e4b5fa84c501f8a784c8f4a0113ea74af2ef4ea840a2a052e33a1f6b3e354761f6966301b00de787bf35a8f259f43f5c6b298ba7ce56f19efec23286c40af589f8d136ba02a4e8c6ed0ec70ae825ed54fcd70fa5acdd75ae8a4d17a34a554fa3b6f3a2482d9462df72ae4cca800cf50f008bb6a&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:48620->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065652Z&X-Goog-Expires=86399&X-Goog-Signature=70e86fac9315b3f1b972a2e965d19160dd68224c29b64e26d1273b1cf0ee0cd71a46b173414db97392275bb488e58c14002348a4e91b03b1a747512a2ab3b841817edc1288805b4177ab9838fb3417705a17bd647a0d9441f2a9d0f1788c3cb687ab8169e5be11b49506785f6ed913ec58a1cabca533c781c70847d44493af27a7f03875ceb771ebef489a0b4e4b5fa84c501f8a784c8f4a0113ea74af2ef4ea840a2a052e33a1f6b3e354761f6966301b00de787bf35a8f259f43f5c6b298ba7ce56f19efec23286c40af589f8d136ba02a4e8c6ed0ec70ae825ed54fcd70fa5acdd75ae8a4d17a34a554fa3b6f3a2482d9462df72ae4cca800cf50f008bb6a&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:06:28.340218333+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=8a95163e0c0f889892ef15b4a88e43d530288eda1d8571066f8e5a5a448332ea60912856e41297390d209d6ef04c28c92498205a84c3ed6ba3f952108c0e9652e9d14ab6e33add6388a6f39baf85d21db8f98a78b2d267a131a011a5d79b267ba86938c2efe0d867337f5cf8a27924e4ca34fda8e2302d0ddbfa5d507e64601856113db1b9ace6fb3ee615f8f3b05ce7a2fe03f38fd8953c9ed405d6a5ff00bd2c59023b984f8a19d6b9653b17a073d109be52afdedc26e456311889d732a252a242a9388117272376c4cec78ffcef9c8f238bea6b854f7189d3e1e721e175292f5da73859ff0d5f0ac7ee2ca45cad4bb3ea441cfb6d3d91f42f9bf287312182&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:57732->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=8a95163e0c0f889892ef15b4a88e43d530288eda1d8571066f8e5a5a448332ea60912856e41297390d209d6ef04c28c92498205a84c3ed6ba3f952108c0e9652e9d14ab6e33add6388a6f39baf85d21db8f98a78b2d267a131a011a5d79b267ba86938c2efe0d867337f5cf8a27924e4ca34fda8e2302d0ddbfa5d507e64601856113db1b9ace6fb3ee615f8f3b05ce7a2fe03f38fd8953c9ed405d6a5ff00bd2c59023b984f8a19d6b9653b17a073d109be52afdedc26e456311889d732a252a242a9388117272376c4cec78ffcef9c8f238bea6b854f7189d3e1e721e175292f5da73859ff0d5f0ac7ee2ca45cad4bb3ea441cfb6d3d91f42f9bf287312182&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:07:58.387973824+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065652Z&X-Goog-Expires=86399&X-Goog-Signature=90ed12f3fe9c9b10f930fadccdb3bf6688307cfcb66aa11a69d0f96047f0ff09401ed55e9eb0cb66cbd9d383845cd893c0156b96247b225f27a027b4dc63458a22e352ddf0db6cfd76d6737e9a3197d7ed8937a9550a5b8a9256d656be8b242e54fde209064a20f89273fa6fcaa8086fd9c43159c4782be5263a45e453d93b8c02e34e41a6371ac8052670756a97db99e0c5e679fd89f8005910eeed307e69a82bdc2998e602effba1d7522c5d65ac371bd161f447c40092e29f017f7bfd56c41ecb3eff9690ce74b62251fecf37078861335e37aa8d57bd4c2325dbc55a727fbc627b2e68156e3e8e7c70da5b4ef4c0032dfaa57f327bc58f069084d184a500&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:47722->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065652Z&X-Goog-Expires=86399&X-Goog-Signature=90ed12f3fe9c9b10f930fadccdb3bf6688307cfcb66aa11a69d0f96047f0ff09401ed55e9eb0cb66cbd9d383845cd893c0156b96247b225f27a027b4dc63458a22e352ddf0db6cfd76d6737e9a3197d7ed8937a9550a5b8a9256d656be8b242e54fde209064a20f89273fa6fcaa8086fd9c43159c4782be5263a45e453d93b8c02e34e41a6371ac8052670756a97db99e0c5e679fd89f8005910eeed307e69a82bdc2998e602effba1d7522c5d65ac371bd161f447c40092e29f017f7bfd56c41ecb3eff9690ce74b62251fecf37078861335e37aa8d57bd4c2325dbc55a727fbc627b2e68156e3e8e7c70da5b4ef4c0032dfaa57f327bc58f069084d184a500&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:08:08.206013764+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065652Z&X-Goog-Expires=86399&X-Goog-Signature=90ed12f3fe9c9b10f930fadccdb3bf6688307cfcb66aa11a69d0f96047f0ff09401ed55e9eb0cb66cbd9d383845cd893c0156b96247b225f27a027b4dc63458a22e352ddf0db6cfd76d6737e9a3197d7ed8937a9550a5b8a9256d656be8b242e54fde209064a20f89273fa6fcaa8086fd9c43159c4782be5263a45e453d93b8c02e34e41a6371ac8052670756a97db99e0c5e679fd89f8005910eeed307e69a82bdc2998e602effba1d7522c5d65ac371bd161f447c40092e29f017f7bfd56c41ecb3eff9690ce74b62251fecf37078861335e37aa8d57bd4c2325dbc55a727fbc627b2e68156e3e8e7c70da5b4ef4c0032dfaa57f327bc58f069084d184a500&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:34432->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065652Z&X-Goog-Expires=86399&X-Goog-Signature=90ed12f3fe9c9b10f930fadccdb3bf6688307cfcb66aa11a69d0f96047f0ff09401ed55e9eb0cb66cbd9d383845cd893c0156b96247b225f27a027b4dc63458a22e352ddf0db6cfd76d6737e9a3197d7ed8937a9550a5b8a9256d656be8b242e54fde209064a20f89273fa6fcaa8086fd9c43159c4782be5263a45e453d93b8c02e34e41a6371ac8052670756a97db99e0c5e679fd89f8005910eeed307e69a82bdc2998e602effba1d7522c5d65ac371bd161f447c40092e29f017f7bfd56c41ecb3eff9690ce74b62251fecf37078861335e37aa8d57bd4c2325dbc55a727fbc627b2e68156e3e8e7c70da5b4ef4c0032dfaa57f327bc58f069084d184a500&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:08:10.671647124+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065657Z&X-Goog-Expires=86399&X-Goog-Signature=b499547cdb15cac87e48fd391f9ed5dad4148a5e06655bd6e6ad06f7feb1fe9ec68cf9977d5655d6f0d89419803ffdf41be6376849cf43a02f5131adf34a050997e1d8d7cb7ab1fd893c7419d00769e0721e52a462ae922282e98feb03c1a136b4f3432c28f72e3a7f0a08efbcd101fb4dfa14e0959db1d1908cebae7ee00098b30a81916a127012f6418d1a91a93cfdc2f330bd80726446f2715bfe93a3117b6c1792f2067565ad2ab643e19967291f785e76d1c2a8d25402682089fc241b6fd8dbc721d8e51fb830fa0a6a83aabe60a39eaf11c1ffd475a406852af5563e870416b290e5e3e50c867afb12812a5a1433d3630064613b00ac1e56e148b48cd5&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:56624->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065657Z&X-Goog-Expires=86399&X-Goog-Signature=b499547cdb15cac87e48fd391f9ed5dad4148a5e06655bd6e6ad06f7feb1fe9ec68cf9977d5655d6f0d89419803ffdf41be6376849cf43a02f5131adf34a050997e1d8d7cb7ab1fd893c7419d00769e0721e52a462ae922282e98feb03c1a136b4f3432c28f72e3a7f0a08efbcd101fb4dfa14e0959db1d1908cebae7ee00098b30a81916a127012f6418d1a91a93cfdc2f330bd80726446f2715bfe93a3117b6c1792f2067565ad2ab643e19967291f785e76d1c2a8d25402682089fc241b6fd8dbc721d8e51fb830fa0a6a83aabe60a39eaf11c1ffd475a406852af5563e870416b290e5e3e50c867afb12812a5a1433d3630064613b00ac1e56e148b48cd5&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:08:27.082693937+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=92ae95a068573a7caa07db2b3bcab253490e0ebe38e0fa761f5a93e49b4c4a5d8e2e667590c2c9e17deb189bbaefcf1b894a2011e61c1124d05ae9a8e9bec1f9d63d6def9d54a24b98f45661ad1a17719ea840a41176dc66ac4ed14242c2d74e4ba8600bdddc171e2f3665abd1a0e289176017c0265fe3c49df53270fd495056a1154d7151533eddfd23de83ffc0ec604a59a24e68e2124c9461b71269bfd51f971767d3402294f3a605f7d2c69730216b6b9e3f14a94e38918150fa9ff89bed4ca0449595c0b2392cbd9fd87b07910c18ceaaf32de694b69ee7d3c9845d3c6ff422849de5ae0bc2018170c0533c41df905d8af0f66ea3ba4713d1810814ca00&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:55834->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065634Z&X-Goog-Expires=86399&X-Goog-Signature=92ae95a068573a7caa07db2b3bcab253490e0ebe38e0fa761f5a93e49b4c4a5d8e2e667590c2c9e17deb189bbaefcf1b894a2011e61c1124d05ae9a8e9bec1f9d63d6def9d54a24b98f45661ad1a17719ea840a41176dc66ac4ed14242c2d74e4ba8600bdddc171e2f3665abd1a0e289176017c0265fe3c49df53270fd495056a1154d7151533eddfd23de83ffc0ec604a59a24e68e2124c9461b71269bfd51f971767d3402294f3a605f7d2c69730216b6b9e3f14a94e38918150fa9ff89bed4ca0449595c0b2392cbd9fd87b07910c18ceaaf32de694b69ee7d3c9845d3c6ff422849de5ae0bc2018170c0533c41df905d8af0f66ea3ba4713d1810814ca00&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:08:27.873237787+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065652Z&X-Goog-Expires=86399&X-Goog-Signature=90ed12f3fe9c9b10f930fadccdb3bf6688307cfcb66aa11a69d0f96047f0ff09401ed55e9eb0cb66cbd9d383845cd893c0156b96247b225f27a027b4dc63458a22e352ddf0db6cfd76d6737e9a3197d7ed8937a9550a5b8a9256d656be8b242e54fde209064a20f89273fa6fcaa8086fd9c43159c4782be5263a45e453d93b8c02e34e41a6371ac8052670756a97db99e0c5e679fd89f8005910eeed307e69a82bdc2998e602effba1d7522c5d65ac371bd161f447c40092e29f017f7bfd56c41ecb3eff9690ce74b62251fecf37078861335e37aa8d57bd4c2325dbc55a727fbc627b2e68156e3e8e7c70da5b4ef4c0032dfaa57f327bc58f069084d184a500&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:36340->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ep0ib6n2/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T065652Z&X-Goog-Expires=86399&X-Goog-Signature=90ed12f3fe9c9b10f930fadccdb3bf6688307cfcb66aa11a69d0f96047f0ff09401ed55e9eb0cb66cbd9d383845cd893c0156b96247b225f27a027b4dc63458a22e352ddf0db6cfd76d6737e9a3197d7ed8937a9550a5b8a9256d656be8b242e54fde209064a20f89273fa6fcaa8086fd9c43159c4782be5263a45e453d93b8c02e34e41a6371ac8052670756a97db99e0c5e679fd89f8005910eeed307e69a82bdc2998e602effba1d7522c5d65ac371bd161f447c40092e29f017f7bfd56c41ecb3eff9690ce74b62251fecf37078861335e37aa8d57bd4c2325dbc55a727fbc627b2e68156e3e8e7c70da5b4ef4c0032dfaa57f327bc58f069084d184a500&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:09:27.468530265+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-08-13T15:09:33.010674188+08:00","level":"INFO","msg":"handler: closed","stream_id":"ep0ib6n2"} +{"time":"2025-08-13T15:09:33.015821202+08:00","level":"INFO","msg":"sender: closed","stream_id":"ep0ib6n2"} +{"time":"2025-08-13T15:09:33.015855038+08:00","level":"INFO","msg":"stream: closed","id":"ep0ib6n2"} diff --git a/BioReason_new/wandb/run-20250813_145627-ep0ib6n2/logs/debug.log b/BioReason_new/wandb/run-20250813_145627-ep0ib6n2/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..a1530f3534abed3cdbe334c8de1c5b655df54b40 --- /dev/null +++ b/BioReason_new/wandb/run-20250813_145627-ep0ib6n2/logs/debug.log @@ -0,0 +1,23 @@ +2025-08-13 14:56:27,097 INFO MainThread:10150 [wandb_setup.py:_flush():80] Current SDK version is 0.21.1 +2025-08-13 14:56:27,098 INFO MainThread:10150 [wandb_setup.py:_flush():80] Configure stats pid to 10150 +2025-08-13 14:56:27,098 INFO MainThread:10150 [wandb_setup.py:_flush():80] Loading settings from /root/.config/wandb/settings +2025-08-13 14:56:27,098 INFO MainThread:10150 [wandb_setup.py:_flush():80] Loading settings from /nas/shared/kilab/wangyujia/BioReason_new/wandb/settings +2025-08-13 14:56:27,098 INFO MainThread:10150 [wandb_setup.py:_flush():80] Loading settings from environment variables +2025-08-13 14:56:27,098 INFO MainThread:10150 [wandb_init.py:setup_run_log_directory():703] Logging user logs to /nas/shared/kilab/wangyujia/BioReason_new/wandb/run-20250813_145627-ep0ib6n2/logs/debug.log +2025-08-13 14:56:27,098 INFO MainThread:10150 [wandb_init.py:setup_run_log_directory():704] Logging internal logs to /nas/shared/kilab/wangyujia/BioReason_new/wandb/run-20250813_145627-ep0ib6n2/logs/debug-internal.log +2025-08-13 14:56:27,098 INFO MainThread:10150 [wandb_init.py:init():830] calling init triggers +2025-08-13 14:56:27,098 INFO MainThread:10150 [wandb_init.py:init():835] wandb.init called with sweep_config: {} +config: {'text_model_name': '/oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged', 'protein_model_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m', 'qformer_model_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft', 'cache_dir': '/model-weights', 'num_query_tokens': 8, 'train_dataset': '/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl', 'valid_dataset': '/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl', 'eval_dataset': True, 'output_dir': './contrastive_outputs', 'num_epochs': 10, 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 0.01, 'warmup_steps': 1000, 'gradient_accumulation_steps': 1, 'temperature': 0.07, 'freeze_protein_model': True, 'freeze_text_model': True, 'protein_weight': 1.0, 'text_weight': 1.0, 'enable_ptm': True, 'ptm_weight': 1.0, 'max_length_protein': 1024, 'max_length_text': 512, 'num_workers': 8, 'logging_steps': 100, 'eval_steps': 500, 'save_steps': 1000, 'save_total_limit': 3, 'fp16': False, 'bf16': False, 'seed': 42, 'use_wandb': True, 'wandb_project': 'protein-llm-contrastive', 'wandb_entity': None, '_wandb': {}} +2025-08-13 14:56:27,098 INFO MainThread:10150 [wandb_init.py:init():871] starting backend +2025-08-13 14:56:27,306 INFO MainThread:10150 [wandb_init.py:init():874] sending inform_init request +2025-08-13 14:56:27,312 INFO MainThread:10150 [wandb_init.py:init():882] backend started and connected +2025-08-13 14:56:27,313 INFO MainThread:10150 [wandb_init.py:init():953] updated telemetry +2025-08-13 14:56:27,341 INFO MainThread:10150 [wandb_init.py:init():977] communicating run to backend with 90.0 second timeout +2025-08-13 14:56:33,672 INFO MainThread:10150 [wandb_init.py:init():1029] starting run threads in backend +2025-08-13 14:56:33,780 INFO MainThread:10150 [wandb_run.py:_console_start():2494] atexit reg +2025-08-13 14:56:33,780 INFO MainThread:10150 [wandb_run.py:_redirect():2342] redirect: wrap_raw +2025-08-13 14:56:33,780 INFO MainThread:10150 [wandb_run.py:_redirect():2411] Wrapping output streams. +2025-08-13 14:56:33,780 INFO MainThread:10150 [wandb_run.py:_redirect():2434] Redirects installed. +2025-08-13 14:56:33,783 INFO MainThread:10150 [wandb_init.py:init():1075] run started, returning control to user process +2025-08-13 14:56:47,035 INFO MainThread:10150 [wandb_run.py:_config_callback():1380] config_cb None None {'output_dir': './contrastive_outputs', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': True, 'do_predict': False, 'eval_strategy': 'steps', 'prediction_loss_only': False, 'per_device_train_batch_size': 8, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'eval_delay': 0, 'torch_empty_cache_steps': None, 'learning_rate': 0.0001, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 10, 'max_steps': -1, 'lr_scheduler_type': 'linear', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.0, 'warmup_steps': 1000, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': './contrastive_outputs/runs/Aug13_14-56-41_dsw-265304-585cc9d768-ckfd7', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 100, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 1000, 'save_total_limit': 3, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'restore_callback_states_from_checkpoint': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 42, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': False, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 500, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': None, 'disable_tqdm': False, 'remove_unused_columns': False, 'label_names': None, 'load_best_model_at_end': True, 'metric_for_best_model': 'eval_avg_recall_at_1', 'greater_is_better': True, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'deepspeed': None, 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['wandb'], 'ddp_find_unused_parameters': False, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '', 'hub_private_repo': None, 'hub_always_push': False, 'hub_revision': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'include_for_metrics': [], 'eval_do_concat_batches': True, 'fp16_backend': 'auto', 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None, 'batch_eval_metrics': False, 'eval_on_start': False, 'use_liger_kernel': False, 'liger_kernel_config': None, 'eval_use_gather_object': False, 'average_tokens_across_devices': False, 'temperature': 0.07, 'freeze_protein_model': True, 'freeze_text_model': True, 'protein_weight': 1.0, 'text_weight': 1.0, 'max_length_protein': 1024, 'max_length_text': 512, 'enable_ptm': True, 'ptm_weight': 1.0} +2025-08-13 14:56:51,693 INFO MsgRouterThr:10150 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles. diff --git a/BioReason_new/wandb/run-20250813_145627-ep0ib6n2/run-ep0ib6n2.wandb b/BioReason_new/wandb/run-20250813_145627-ep0ib6n2/run-ep0ib6n2.wandb new file mode 100644 index 0000000000000000000000000000000000000000..3cd9278b158117c5a0bfc8b9942a06e3c7fba956 Binary files /dev/null and b/BioReason_new/wandb/run-20250813_145627-ep0ib6n2/run-ep0ib6n2.wandb differ diff --git a/BioReason_new/wandb/run-20250813_151502-efaj8qxg/files/config.yaml b/BioReason_new/wandb/run-20250813_151502-efaj8qxg/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0b9e8fecf8a103c7a755cc2ab2ee648cc6981f4c --- /dev/null +++ b/BioReason_new/wandb/run-20250813_151502-efaj8qxg/files/config.yaml @@ -0,0 +1,457 @@ +_wandb: + value: + cli_version: 0.21.1 + e: + wof9vn4farxkwiz68rqknt7azap7s5wq: + args: + - --text_model_name + - /oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged + - --protein_model_name + - /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m + - --qformer_model_name + - /nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft + - --num_query_tokens + - "8" + - --train_dataset + - /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl + - --valid_dataset + - /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl + - --output_dir + - ./contrastive_outputs + - --num_epochs + - "10" + - --batch_size + - "8" + - --learning_rate + - "1e-4" + - --temperature + - "0.07" + - --freeze_protein_model + - --freeze_text_model + - --enable_ptm + - --max_length_protein + - "1024" + - --max_length_text + - "512" + - --num_workers + - "8" + - --eval_dataset + - --use_wandb + - --wandb_project + - protein-llm-contrastive + - --logging_steps + - "100" + - --eval_steps + - "500" + - --save_steps + - "1000" + codePath: wangyujia/BioReason_new/train_contrastive.py + codePathLocal: train_contrastive.py + cpu_count: 64 + cpu_count_logical: 64 + cudaVersion: "12.1" + disk: + /: + total: "1623302262784" + used: "1015259136" + executable: /root/miniconda3/envs/bioreason/bin/python + git: + commit: b8caf406aa1699c788f0ca6e44a1769452c317db + remote: https://github.com/PorUna-byte/PAR.git + gpu: NVIDIA A800-SXM4-80GB + gpu_count: 8 + gpu_nvidia: + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-62fcc532-880f-7324-eb13-3b739a38c29b + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-af1197aa-3e7d-a53e-185d-04b1d64164da + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-3e6e551f-2d7d-a27b-654a-da56933d2283 + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-a947b64a-c545-ee5e-30e6-b4b0e2dd8fdc + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-ebd04f3f-c437-1b8b-26f2-4b1ebfee93c4 + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-53994891-72a6-25be-a4c8-e32c695155b0 + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-a927d00a-15c5-31f5-9ae1-9d2e773821a1 + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-93d555c4-29a0-cd78-abac-d98cd160d4e1 + host: dsw-265304-585cc9d768-ckfd7 + memory: + total: "549755813888" + os: Linux-5.10.134-008.18.kangaroo.al8.x86_64-x86_64-with-glibc2.35 + program: /nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py + python: CPython 3.11.0 + root: /nas/shared/kilab/wangyujia/BioReason_new + startedAt: "2025-08-13T07:15:02.581009Z" + writerId: wof9vn4farxkwiz68rqknt7azap7s5wq + m: + - "1": train/global_step + "6": + - 3 + "7": [] + - "2": '*' + "5": 1 + "6": + - 1 + "7": [] + python_version: 3.11.0 + t: + "1": + - 1 + - 9 + - 11 + - 41 + - 49 + - 51 + - 71 + - 84 + - 98 + - 103 + "2": + - 1 + - 9 + - 11 + - 41 + - 49 + - 51 + - 71 + - 84 + - 98 + - 103 + "3": + - 7 + - 13 + - 16 + - 66 + "4": 3.11.0 + "5": 0.21.1 + "6": 4.55.0 + "9": + "1": transformers_trainer + "12": 0.21.1 + "13": linux-x86_64 +accelerator_config: + value: + dispatch_batches: null + even_batches: true + gradient_accumulation_kwargs: null + non_blocking: false + split_batches: false + use_seedable_sampler: true +adafactor: + value: false +adam_beta1: + value: 0.9 +adam_beta2: + value: 0.999 +adam_epsilon: + value: 1e-08 +auto_find_batch_size: + value: false +average_tokens_across_devices: + value: false +batch_eval_metrics: + value: false +batch_size: + value: 8 +bf16: + value: false +bf16_full_eval: + value: false +cache_dir: + value: /model-weights +data_seed: + value: null +dataloader_drop_last: + value: false +dataloader_num_workers: + value: 8 +dataloader_persistent_workers: + value: false +dataloader_pin_memory: + value: true +dataloader_prefetch_factor: + value: null +ddp_backend: + value: null +ddp_broadcast_buffers: + value: null +ddp_bucket_cap_mb: + value: null +ddp_find_unused_parameters: + value: false +ddp_timeout: + value: 1800 +debug: + value: [] +deepspeed: + value: null +disable_tqdm: + value: false +do_eval: + value: true +do_predict: + value: false +do_train: + value: false +enable_ptm: + value: true +eval_accumulation_steps: + value: null +eval_dataset: + value: true +eval_delay: + value: 0 +eval_do_concat_batches: + value: true +eval_on_start: + value: false +eval_steps: + value: 500 +eval_strategy: + value: steps +eval_use_gather_object: + value: false +fp16: + value: false +fp16_backend: + value: auto +fp16_full_eval: + value: false +fp16_opt_level: + value: O1 +freeze_protein_model: + value: true +freeze_text_model: + value: true +fsdp: + value: [] +fsdp_config: + value: + min_num_params: 0 + xla: false + xla_fsdp_grad_ckpt: false + xla_fsdp_v2: false +fsdp_min_num_params: + value: 0 +fsdp_transformer_layer_cls_to_wrap: + value: null +full_determinism: + value: false +gradient_accumulation_steps: + value: 1 +gradient_checkpointing: + value: false +gradient_checkpointing_kwargs: + value: null +greater_is_better: + value: true +group_by_length: + value: false +half_precision_backend: + value: auto +hub_always_push: + value: false +hub_model_id: + value: null +hub_private_repo: + value: null +hub_revision: + value: null +hub_strategy: + value: every_save +hub_token: + value: +ignore_data_skip: + value: false +include_for_metrics: + value: [] +include_inputs_for_metrics: + value: false +include_num_input_tokens_seen: + value: false +include_tokens_per_second: + value: false +jit_mode_eval: + value: false +label_names: + value: null +label_smoothing_factor: + value: 0 +learning_rate: + value: 0.0001 +length_column_name: + value: length +liger_kernel_config: + value: null +load_best_model_at_end: + value: true +local_rank: + value: 0 +log_level: + value: passive +log_level_replica: + value: warning +log_on_each_node: + value: true +logging_dir: + value: ./contrastive_outputs/runs/Aug13_15-15-23_dsw-265304-585cc9d768-ckfd7 +logging_first_step: + value: false +logging_nan_inf_filter: + value: true +logging_steps: + value: 100 +logging_strategy: + value: steps +lr_scheduler_type: + value: linear +max_grad_norm: + value: 1 +max_length_protein: + value: 1024 +max_length_text: + value: 512 +max_steps: + value: -1 +metric_for_best_model: + value: eval_avg_recall_at_1 +mp_parameters: + value: "" +neftune_noise_alpha: + value: null +no_cuda: + value: false +num_epochs: + value: 10 +num_query_tokens: + value: 8 +num_train_epochs: + value: 10 +num_workers: + value: 8 +optim: + value: adamw_torch +optim_args: + value: null +optim_target_modules: + value: null +output_dir: + value: ./contrastive_outputs +overwrite_output_dir: + value: false +past_index: + value: -1 +per_device_eval_batch_size: + value: 8 +per_device_train_batch_size: + value: 8 +per_gpu_eval_batch_size: + value: null +per_gpu_train_batch_size: + value: null +prediction_loss_only: + value: false +protein_model_name: + value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m +protein_weight: + value: 1 +ptm_weight: + value: 1 +push_to_hub: + value: false +push_to_hub_model_id: + value: null +push_to_hub_organization: + value: null +push_to_hub_token: + value: +qformer_model_name: + value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft +ray_scope: + value: last +remove_unused_columns: + value: false +report_to: + value: + - wandb +restore_callback_states_from_checkpoint: + value: false +resume_from_checkpoint: + value: null +run_name: + value: null +save_on_each_node: + value: false +save_only_model: + value: false +save_safetensors: + value: true +save_steps: + value: 1000 +save_strategy: + value: steps +save_total_limit: + value: 3 +seed: + value: 42 +skip_memory_metrics: + value: true +temperature: + value: 0.07 +text_model_name: + value: /oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged +text_weight: + value: 1 +tf32: + value: null +torch_compile: + value: false +torch_compile_backend: + value: null +torch_compile_mode: + value: null +torch_empty_cache_steps: + value: null +torchdynamo: + value: null +tpu_metrics_debug: + value: false +tpu_num_cores: + value: null +train_dataset: + value: /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl +use_cpu: + value: false +use_ipex: + value: false +use_legacy_prediction_loop: + value: false +use_liger_kernel: + value: false +use_mps_device: + value: false +use_wandb: + value: true +valid_dataset: + value: /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl +wandb_entity: + value: null +wandb_project: + value: protein-llm-contrastive +warmup_ratio: + value: 0 +warmup_steps: + value: 1000 +weight_decay: + value: 0.01 diff --git a/BioReason_new/wandb/run-20250813_151502-efaj8qxg/files/output.log b/BioReason_new/wandb/run-20250813_151502-efaj8qxg/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..62c077f0d2437f6d6285dfb5c861717068612a01 --- /dev/null +++ b/BioReason_new/wandb/run-20250813_151502-efaj8qxg/files/output.log @@ -0,0 +1,45 @@ +Loading model... +Loading checkpoint shards: 100%|██████████████████████████████████████████████████████| 4/4 [00:01<00:00, 2.45it/s] +Some weights of EsmModel were not initialized from the model checkpoint at /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m and are newly initialized: ['pooler.dense.bias', 'pooler.dense.weight'] +You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference. +Loading datasets... +Training dataset size: 430595 +Eval dataset size: 10000 +Initializing enhanced trainer... +[2025-08-13 15:15:27,683] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) +/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/_distutils_hack/__init__.py:53: UserWarning: Reliance on distutils from stdlib is deprecated. Users must rely on setuptools to provide the distutils module. Avoid importing distutils or import setuptools first, and avoid setting SETUPTOOLS_USE_DISTUTILS=stdlib. Register concerns at https://github.com/pypa/setuptools/issues/new?template=distutils-deprecation.yml + warnings.warn( +[2025-08-13 15:15:28,672] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False +Starting enhanced contrastive training... +- Contrastive learning enabled +- Protein-text matching: enabled +- Temperature: 0.07 +- PTM weight: 1.0 + 0%| | 0/67290 [00:00 + trainer = main(args) + ^^^^^^^^^^ + File "/nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py", line 399, in main + trainer.train() + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/trainer.py", line 2238, in train + return inner_training_loop( + ^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/trainer.py", line 2582, in _inner_training_loop + tr_loss_step = self.training_step(model, inputs, num_items_in_batch) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/trainer.py", line 3796, in training_step + loss = self.compute_loss(model, inputs, num_items_in_batch=num_items_in_batch) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/nas/shared/kilab/wangyujia/BioReason_new/bioreason/trainer/contrast_trainer_new.py", line 330, in compute_loss + loss_ptm = self.contrastive_loss.compute_ptm_loss( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/nas/shared/kilab/wangyujia/BioReason_new/bioreason/trainer/contrast_trainer_new.py", line 196, in compute_ptm_loss + output_ptm = qformer( + ^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1762, in _call_impl + return forward_call(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +TypeError: BertModel.forward() got an unexpected keyword argument 'query_embeds' diff --git a/BioReason_new/wandb/run-20250813_151502-efaj8qxg/files/requirements.txt b/BioReason_new/wandb/run-20250813_151502-efaj8qxg/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..743d03fe27e0b7e200935023c07a1d7fa61b99ce --- /dev/null +++ b/BioReason_new/wandb/run-20250813_151502-efaj8qxg/files/requirements.txt @@ -0,0 +1,233 @@ +jupyterlab_pygments==0.3.0 +depyf==0.19.0 +av==15.0.0 +fastapi==0.116.1 +ray==2.48.0 +cachetools==6.1.0 +prometheus_client==0.22.1 +requests==2.32.4 +jupyter-lsp==2.2.6 +jupyter-events==0.12.0 +anyio==4.10.0 +pybase64==1.4.2 +fastjsonschema==2.21.1 +cloudpickle==3.1.1 +sentry-sdk==2.34.1 +torchmetrics==1.8.1 +wheel==0.45.1 +idna==3.10 +defusedxml==0.7.1 +starlette==0.47.2 +python-dotenv==1.1.1 +comm==0.2.3 +multidict==6.6.3 +python-json-logger==3.3.0 +jupyter_server==2.16.0 +nvidia-nvtx-cu12==12.6.77 +websockets==15.0.1 +referencing==0.36.2 +gguf==0.17.1 +lm-format-enforcer==0.10.12 +xformers==0.0.31 +ninja==1.11.1.4 +beautifulsoup4==4.13.4 +dnspython==2.7.0 +certifi==2025.8.3 +markdown-it-py==3.0.0 +stack-data==0.6.3 +nvidia-cusparselt-cu12==0.6.3 +networkx==3.5 +aiohappyeyeballs==2.6.1 +nvidia-cuda-cupti-cu12==12.6.80 +llguidance==0.7.30 +rich==14.1.0 +outlines_core==0.2.10 +nvidia-nccl-cu12==2.26.2 +pandas==2.3.1 +pydantic_core==2.33.2 +argon2-cffi==25.1.0 +babel==2.17.0 +isoduration==20.11.0 +uvloop==0.21.0 +json5==0.12.0 +widgetsnbextension==4.0.14 +aiohttp==3.12.15 +notebook_shim==0.2.4 +pandocfilters==1.5.1 +vllm==0.10.0 +blake3==1.0.5 +diskcache==5.6.3 +h11==0.16.0 +attrs==25.3.0 +nvidia-cusolver-cu12==11.7.1.2 +click==8.2.1 +cupy-cuda12x==13.5.1 +jedi==0.19.2 +py-cpuinfo==9.0.0 +overrides==7.7.0 +pytz==2025.2 +protobuf==6.31.1 +dill==0.3.8 +ipykernel==6.30.1 +frozenlist==1.7.0 +huggingface-hub==0.34.4 +rignore==0.6.4 +mistune==3.1.3 +sniffio==1.3.1 +httpx==0.28.1 +sentencepiece==0.2.0 +typing_extensions==4.14.1 +typing-inspection==0.4.1 +async-lru==2.0.5 +bioreason==0.1.0 +charset-normalizer==3.4.3 +llvmlite==0.44.0 +safetensors==0.6.2 +fqdn==1.5.1 +peft==0.17.0 +fastapi-cli==0.0.8 +torch==2.7.1 +jsonschema==4.25.0 +torchvision==0.22.1 +parso==0.8.4 +pillow==11.3.0 +pycountry==24.6.1 +scipy==1.16.1 +wandb==0.21.1 +pip==25.1 +wcwidth==0.2.13 +mistral_common==1.8.3 +nvidia-cudnn-cu12==9.5.1.17 +asttokens==3.0.0 +numpy==2.2.6 +uvicorn==0.35.0 +jupyter_client==8.6.3 +debugpy==1.8.16 +fastapi-cloud-cli==0.1.5 +compressed-tensors==0.10.2 +GitPython==3.1.45 +nvidia-ml-py==13.580.65 +tiktoken==0.11.0 +partial-json-parser==0.2.1.1.post6 +prompt_toolkit==3.0.51 +uri-template==1.3.0 +pyarrow==21.0.0 +nvidia-cuda-nvrtc-cu12==12.6.77 +propcache==0.3.2 +matplotlib-inline==0.1.7 +pure_eval==0.2.3 +nest-asyncio==1.6.0 +deepspeed==0.17.4 +einops==0.8.1 +triton==3.3.1 +trl==0.21.0 +pyzmq==27.0.1 +httptools==0.6.4 +ipython==9.4.0 +packaging==25.0 +hjson==3.1.0 +soxr==0.5.0.post1 +webcolors==24.11.1 +PyYAML==6.0.2 +ipywidgets==8.1.7 +lark==1.2.2 +python-multipart==0.0.20 +tinycss2==1.4.0 +mpmath==1.3.0 +psutil==7.0.0 +nbformat==5.10.4 +qwen-vl-utils==0.0.11 +notebook==7.4.5 +jupyterlab==4.4.5 +transformers==4.55.0 +urllib3==2.5.0 +terminado==0.18.1 +xgrammar==0.1.21 +xxhash==3.5.0 +tzdata==2025.2 +jupyterlab_widgets==3.0.15 +aiosignal==1.4.0 +annotated-types==0.7.0 +nvidia-nvjitlink-cu12==12.6.85 +httpcore==1.0.9 +pexpect==4.9.0 +tornado==6.5.2 +regex==2025.7.33 +filelock==3.18.0 +nvidia-cuda-runtime-cu12==12.6.77 +openai==1.90.0 +Pygments==2.19.2 +cffi==1.17.1 +nvidia-cufft-cu12==11.3.0.4 +rpds-py==0.27.0 +rich-toolkit==0.15.0 +bleach==6.2.0 +sympy==1.14.0 +python-dateutil==2.9.0.post0 +watchfiles==1.1.0 +nbclient==0.10.2 +hf-xet==1.1.7 +Send2Trash==1.8.3 +smmap==5.0.2 +tokenizers==0.21.4 +yarl==1.20.1 +MarkupSafe==3.0.2 +prometheus-fastapi-instrumentator==7.1.0 +rfc3986-validator==0.1.1 +bitsandbytes==0.46.1 +nvidia-cufile-cu12==1.11.1.6 +fsspec==2025.3.0 +executing==2.2.0 +pydantic==2.11.7 +jupyter_core==5.8.1 +msgspec==0.19.0 +nvidia-cusparse-cu12==12.5.4.2 +opencv-python-headless==4.12.0.88 +webencodings==0.5.1 +ptyprocess==0.7.0 +pytorch-lightning==2.5.2 +Jinja2==3.1.6 +websocket-client==1.8.0 +jsonpointer==3.0.0 +traitlets==5.14.3 +numba==0.61.2 +soupsieve==2.7 +cbor2==5.6.5 +multiprocess==0.70.16 +email_validator==2.2.0 +distro==1.9.0 +accelerate==1.10.0 +torchaudio==2.7.1 +jiter==0.10.0 +rfc3339-validator==0.1.4 +jupyter==1.1.1 +typer==0.16.0 +gitdb==4.0.12 +types-python-dateutil==2.9.0.20250809 +soundfile==0.13.1 +astor==0.8.1 +shellingham==1.5.4 +jupyterlab_server==2.27.3 +nvidia-curand-cu12==10.3.7.77 +jupyter-console==6.6.3 +msgpack==1.1.1 +six==1.17.0 +pycparser==2.22 +arrow==1.3.0 +tqdm==4.67.1 +lightning-utilities==0.15.2 +interegular==0.3.3 +rfc3987-syntax==1.1.0 +ipython_pygments_lexers==1.1.1 +datasets==4.0.0 +jupyter_server_terminals==0.5.3 +platformdirs==4.3.8 +pydantic-extra-types==2.10.5 +decorator==5.2.1 +mdurl==0.1.2 +nvidia-cublas-cu12==12.6.4.1 +jsonschema-specifications==2025.4.1 +setuptools==78.1.1 +fastrlock==0.8.3 +argon2-cffi-bindings==25.1.0 +nbconvert==7.16.6 diff --git a/BioReason_new/wandb/run-20250813_151502-efaj8qxg/files/wandb-metadata.json b/BioReason_new/wandb/run-20250813_151502-efaj8qxg/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..39de9bde13b7d15b7cbf5f64150c996346526f3b --- /dev/null +++ b/BioReason_new/wandb/run-20250813_151502-efaj8qxg/files/wandb-metadata.json @@ -0,0 +1,115 @@ +{ + "os": "Linux-5.10.134-008.18.kangaroo.al8.x86_64-x86_64-with-glibc2.35", + "python": "CPython 3.11.0", + "startedAt": "2025-08-13T07:15:02.581009Z", + "args": [ + "--text_model_name", + "/oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged", + "--protein_model_name", + "/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m", + "--qformer_model_name", + "/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft", + "--num_query_tokens", + "8", + "--train_dataset", + "/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl", + "--valid_dataset", + "/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl", + "--output_dir", + "./contrastive_outputs", + "--num_epochs", + "10", + "--batch_size", + "8", + "--learning_rate", + "1e-4", + "--temperature", + "0.07", + "--freeze_protein_model", + "--freeze_text_model", + "--enable_ptm", + "--max_length_protein", + "1024", + "--max_length_text", + "512", + "--num_workers", + "8", + "--eval_dataset", + "--use_wandb", + "--wandb_project", + "protein-llm-contrastive", + "--logging_steps", + "100", + "--eval_steps", + "500", + "--save_steps", + "1000" + ], + "program": "/nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py", + "codePath": "wangyujia/BioReason_new/train_contrastive.py", + "codePathLocal": "train_contrastive.py", + "git": { + "remote": "https://github.com/PorUna-byte/PAR.git", + "commit": "b8caf406aa1699c788f0ca6e44a1769452c317db" + }, + "root": "/nas/shared/kilab/wangyujia/BioReason_new", + "host": "dsw-265304-585cc9d768-ckfd7", + "executable": "/root/miniconda3/envs/bioreason/bin/python", + "cpu_count": 64, + "cpu_count_logical": 64, + "gpu": "NVIDIA A800-SXM4-80GB", + "gpu_count": 8, + "disk": { + "/": { + "total": "1623302262784", + "used": "1015259136" + } + }, + "memory": { + "total": "549755813888" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-62fcc532-880f-7324-eb13-3b739a38c29b" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-af1197aa-3e7d-a53e-185d-04b1d64164da" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-3e6e551f-2d7d-a27b-654a-da56933d2283" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-a947b64a-c545-ee5e-30e6-b4b0e2dd8fdc" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-ebd04f3f-c437-1b8b-26f2-4b1ebfee93c4" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-53994891-72a6-25be-a4c8-e32c695155b0" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-a927d00a-15c5-31f5-9ae1-9d2e773821a1" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-93d555c4-29a0-cd78-abac-d98cd160d4e1" + } + ], + "cudaVersion": "12.1", + "writerId": "wof9vn4farxkwiz68rqknt7azap7s5wq" +} \ No newline at end of file diff --git a/BioReason_new/wandb/run-20250813_151502-efaj8qxg/files/wandb-summary.json b/BioReason_new/wandb/run-20250813_151502-efaj8qxg/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..b83728ff622654da73d628f469e3fbb77e031a30 --- /dev/null +++ b/BioReason_new/wandb/run-20250813_151502-efaj8qxg/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":14},"_runtime":14} \ No newline at end of file diff --git a/BioReason_new/wandb/run-20250813_151502-efaj8qxg/logs/debug-internal.log b/BioReason_new/wandb/run-20250813_151502-efaj8qxg/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..27156abeeb4fc7139a6ea74eb2a72a08831f7219 --- /dev/null +++ b/BioReason_new/wandb/run-20250813_151502-efaj8qxg/logs/debug-internal.log @@ -0,0 +1,32 @@ +{"time":"2025-08-13T15:15:02.888008292+08:00","level":"INFO","msg":"stream: starting","core version":"0.21.1"} +{"time":"2025-08-13T15:15:15.178738926+08:00","level":"INFO","msg":"stream: created new stream","id":"efaj8qxg"} +{"time":"2025-08-13T15:15:15.179466024+08:00","level":"INFO","msg":"stream: started","id":"efaj8qxg"} +{"time":"2025-08-13T15:15:15.179500051+08:00","level":"INFO","msg":"writer: started","stream_id":"efaj8qxg"} +{"time":"2025-08-13T15:15:15.179516518+08:00","level":"INFO","msg":"sender: started","stream_id":"efaj8qxg"} +{"time":"2025-08-13T15:15:15.179546609+08:00","level":"INFO","msg":"handler: started","stream_id":"efaj8qxg"} +{"time":"2025-08-13T15:15:20.355009716+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/efaj8qxg/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T071519Z&X-Goog-Expires=86399&X-Goog-Signature=88d460d973e6f91da1a5744f00dec3b92c44067bb213951e7663b743587b52235a7e5f404473dda9615fdd1c8c17d459f36fda57d92ad6d3fad1400dbbcedd45729b477d38605d72354bd45bc31c74923a294114d9c80d818cb864db820f1dc188b9ad3eca4b38e243d004e395a907a874430d1b198fbb63cbdf860803c3bbc91cdaac28bc276bc115603a7a46347c7209bee61f5973744af3abadd4a25e49b513e10b15daf485b9585cfdd9f0822b487ed962a76840936c0d0efea22bca250ea956116c19515bc6c696cb1b7b0df2108335785e9416ebe0f652036f1e7e852633ef7296d65a0d88cf18dbbd22e505bff357337f44fe8d1f57cd226ea45c2d1f&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:54362->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/efaj8qxg/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T071519Z&X-Goog-Expires=86399&X-Goog-Signature=88d460d973e6f91da1a5744f00dec3b92c44067bb213951e7663b743587b52235a7e5f404473dda9615fdd1c8c17d459f36fda57d92ad6d3fad1400dbbcedd45729b477d38605d72354bd45bc31c74923a294114d9c80d818cb864db820f1dc188b9ad3eca4b38e243d004e395a907a874430d1b198fbb63cbdf860803c3bbc91cdaac28bc276bc115603a7a46347c7209bee61f5973744af3abadd4a25e49b513e10b15daf485b9585cfdd9f0822b487ed962a76840936c0d0efea22bca250ea956116c19515bc6c696cb1b7b0df2108335785e9416ebe0f652036f1e7e852633ef7296d65a0d88cf18dbbd22e505bff357337f44fe8d1f57cd226ea45c2d1f&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:15:22.943716509+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/efaj8qxg/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T071519Z&X-Goog-Expires=86399&X-Goog-Signature=88d460d973e6f91da1a5744f00dec3b92c44067bb213951e7663b743587b52235a7e5f404473dda9615fdd1c8c17d459f36fda57d92ad6d3fad1400dbbcedd45729b477d38605d72354bd45bc31c74923a294114d9c80d818cb864db820f1dc188b9ad3eca4b38e243d004e395a907a874430d1b198fbb63cbdf860803c3bbc91cdaac28bc276bc115603a7a46347c7209bee61f5973744af3abadd4a25e49b513e10b15daf485b9585cfdd9f0822b487ed962a76840936c0d0efea22bca250ea956116c19515bc6c696cb1b7b0df2108335785e9416ebe0f652036f1e7e852633ef7296d65a0d88cf18dbbd22e505bff357337f44fe8d1f57cd226ea45c2d1f&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:54376->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/efaj8qxg/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T071519Z&X-Goog-Expires=86399&X-Goog-Signature=88d460d973e6f91da1a5744f00dec3b92c44067bb213951e7663b743587b52235a7e5f404473dda9615fdd1c8c17d459f36fda57d92ad6d3fad1400dbbcedd45729b477d38605d72354bd45bc31c74923a294114d9c80d818cb864db820f1dc188b9ad3eca4b38e243d004e395a907a874430d1b198fbb63cbdf860803c3bbc91cdaac28bc276bc115603a7a46347c7209bee61f5973744af3abadd4a25e49b513e10b15daf485b9585cfdd9f0822b487ed962a76840936c0d0efea22bca250ea956116c19515bc6c696cb1b7b0df2108335785e9416ebe0f652036f1e7e852633ef7296d65a0d88cf18dbbd22e505bff357337f44fe8d1f57cd226ea45c2d1f&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:15:26.638241678+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/efaj8qxg/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T071519Z&X-Goog-Expires=86399&X-Goog-Signature=77515741d3d6a1aa2328f782beb0f48c5167fab6c1266cc5c3a77927d9a89f957b020839fd39de588a5b6e79475a413b53706b3a9dab3039b39000be838c59e1f12fdaaf62bbfbe3fe6698a94a94ed82ca2a461185e513aced2840a000ae77be7140d01a0c944aa1c479631d4195efd42fa935568da32d1a59557bcbd2bffa388928d3811fd59421e339bdff37df9c6075922c4f1e9e24a5a137869eae246fb8658dce98d7b0d331ec6276e324eb033931eadb77ac6517941430d91bdb92b9e27ce3035e5b48f8ca176af529342c13a979efc57dfdd6cfa80e13a5049367a476f0c134de70039d3211b70746a04f0626babfa66d2ec8cfe976ffbd6d91d9d006&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:54378->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/efaj8qxg/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T071519Z&X-Goog-Expires=86399&X-Goog-Signature=77515741d3d6a1aa2328f782beb0f48c5167fab6c1266cc5c3a77927d9a89f957b020839fd39de588a5b6e79475a413b53706b3a9dab3039b39000be838c59e1f12fdaaf62bbfbe3fe6698a94a94ed82ca2a461185e513aced2840a000ae77be7140d01a0c944aa1c479631d4195efd42fa935568da32d1a59557bcbd2bffa388928d3811fd59421e339bdff37df9c6075922c4f1e9e24a5a137869eae246fb8658dce98d7b0d331ec6276e324eb033931eadb77ac6517941430d91bdb92b9e27ce3035e5b48f8ca176af529342c13a979efc57dfdd6cfa80e13a5049367a476f0c134de70039d3211b70746a04f0626babfa66d2ec8cfe976ffbd6d91d9d006&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:15:27.91168269+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/efaj8qxg/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T071519Z&X-Goog-Expires=86399&X-Goog-Signature=88d460d973e6f91da1a5744f00dec3b92c44067bb213951e7663b743587b52235a7e5f404473dda9615fdd1c8c17d459f36fda57d92ad6d3fad1400dbbcedd45729b477d38605d72354bd45bc31c74923a294114d9c80d818cb864db820f1dc188b9ad3eca4b38e243d004e395a907a874430d1b198fbb63cbdf860803c3bbc91cdaac28bc276bc115603a7a46347c7209bee61f5973744af3abadd4a25e49b513e10b15daf485b9585cfdd9f0822b487ed962a76840936c0d0efea22bca250ea956116c19515bc6c696cb1b7b0df2108335785e9416ebe0f652036f1e7e852633ef7296d65a0d88cf18dbbd22e505bff357337f44fe8d1f57cd226ea45c2d1f&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:59448->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/efaj8qxg/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T071519Z&X-Goog-Expires=86399&X-Goog-Signature=88d460d973e6f91da1a5744f00dec3b92c44067bb213951e7663b743587b52235a7e5f404473dda9615fdd1c8c17d459f36fda57d92ad6d3fad1400dbbcedd45729b477d38605d72354bd45bc31c74923a294114d9c80d818cb864db820f1dc188b9ad3eca4b38e243d004e395a907a874430d1b198fbb63cbdf860803c3bbc91cdaac28bc276bc115603a7a46347c7209bee61f5973744af3abadd4a25e49b513e10b15daf485b9585cfdd9f0822b487ed962a76840936c0d0efea22bca250ea956116c19515bc6c696cb1b7b0df2108335785e9416ebe0f652036f1e7e852633ef7296d65a0d88cf18dbbd22e505bff357337f44fe8d1f57cd226ea45c2d1f&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:15:33.213087342+08:00","level":"INFO","msg":"stream: closing","id":"efaj8qxg"} +{"time":"2025-08-13T15:15:34.464410098+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/efaj8qxg/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T071533Z&X-Goog-Expires=86399&X-Goog-Signature=9a05e89c850dc41833db428d1fdf4d303a95501e1a60518ddbb1761d0f33be57bb26a9c84ffbe52221822fa259f3a5f21e797ce8c8783413cb8c14f3c8cb86a0acfa17940b8844ad19bb8031ff76c9ab8e917f2d7870eaec1c4bcbc963c5f3a52b8045250f62e9e21a091665e67e3d3ede895755b6d98d9aa4e6783c5f99a7c2e54d3d08392a260a10500cb76bb1cf8942363c945b7008baaa026e138a7036df08d9658699576eb3c47ac64c60246685210f0d80b6ea42325eaf9a9727cf61c6d865980e0b686eb45caf6f96c638dad3b160d874cbbd341c39d9c5b4c484f908649cb5e3b7e06b48f8d5d27ff7021d218df9a1be44f23d9e8e13937d33e27563&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:48556->142.250.73.155:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/efaj8qxg/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T071533Z&X-Goog-Expires=86399&X-Goog-Signature=9a05e89c850dc41833db428d1fdf4d303a95501e1a60518ddbb1761d0f33be57bb26a9c84ffbe52221822fa259f3a5f21e797ce8c8783413cb8c14f3c8cb86a0acfa17940b8844ad19bb8031ff76c9ab8e917f2d7870eaec1c4bcbc963c5f3a52b8045250f62e9e21a091665e67e3d3ede895755b6d98d9aa4e6783c5f99a7c2e54d3d08392a260a10500cb76bb1cf8942363c945b7008baaa026e138a7036df08d9658699576eb3c47ac64c60246685210f0d80b6ea42325eaf9a9727cf61c6d865980e0b686eb45caf6f96c638dad3b160d874cbbd341c39d9c5b4c484f908649cb5e3b7e06b48f8d5d27ff7021d218df9a1be44f23d9e8e13937d33e27563&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:15:34.486339396+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/efaj8qxg/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T071533Z&X-Goog-Expires=86399&X-Goog-Signature=728b16e5ae45a557d355aca5d47a1f22a749044e5f0894f502ef5cb272aaa29f6019d3c2a2f8b2740ab94f241db698c1d657d871c1d6bb4cb25559fcc4e00da68a9fb13eeb707c463237c86932d3a79fef4d3edffdf8131181058584c6020341ae70ea30517affe9cd275076f7d88080e1dec3a9e3aa0ce70f6106272707976fe828f31a585b11218b205b2ea368c257d9dcf6da21893a76a1faf06c772f07df6d06aa1677a1d8c0823cd05b0054356c8b4daa4d575ed6faaca834c60a6f4b07b21b9e8ae07bca84f90ffd581a95209aab1f7b52d84f096f1348383e9fd7871890421f633299ece7e32cc66d05c8f171a7ecccf16756f013b97c35e64bceffe0&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:59450->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/efaj8qxg/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T071533Z&X-Goog-Expires=86399&X-Goog-Signature=728b16e5ae45a557d355aca5d47a1f22a749044e5f0894f502ef5cb272aaa29f6019d3c2a2f8b2740ab94f241db698c1d657d871c1d6bb4cb25559fcc4e00da68a9fb13eeb707c463237c86932d3a79fef4d3edffdf8131181058584c6020341ae70ea30517affe9cd275076f7d88080e1dec3a9e3aa0ce70f6106272707976fe828f31a585b11218b205b2ea368c257d9dcf6da21893a76a1faf06c772f07df6d06aa1677a1d8c0823cd05b0054356c8b4daa4d575ed6faaca834c60a6f4b07b21b9e8ae07bca84f90ffd581a95209aab1f7b52d84f096f1348383e9fd7871890421f633299ece7e32cc66d05c8f171a7ecccf16756f013b97c35e64bceffe0&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:15:36.792451376+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/efaj8qxg/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T071533Z&X-Goog-Expires=86399&X-Goog-Signature=9a05e89c850dc41833db428d1fdf4d303a95501e1a60518ddbb1761d0f33be57bb26a9c84ffbe52221822fa259f3a5f21e797ce8c8783413cb8c14f3c8cb86a0acfa17940b8844ad19bb8031ff76c9ab8e917f2d7870eaec1c4bcbc963c5f3a52b8045250f62e9e21a091665e67e3d3ede895755b6d98d9aa4e6783c5f99a7c2e54d3d08392a260a10500cb76bb1cf8942363c945b7008baaa026e138a7036df08d9658699576eb3c47ac64c60246685210f0d80b6ea42325eaf9a9727cf61c6d865980e0b686eb45caf6f96c638dad3b160d874cbbd341c39d9c5b4c484f908649cb5e3b7e06b48f8d5d27ff7021d218df9a1be44f23d9e8e13937d33e27563&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:59456->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/efaj8qxg/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T071533Z&X-Goog-Expires=86399&X-Goog-Signature=9a05e89c850dc41833db428d1fdf4d303a95501e1a60518ddbb1761d0f33be57bb26a9c84ffbe52221822fa259f3a5f21e797ce8c8783413cb8c14f3c8cb86a0acfa17940b8844ad19bb8031ff76c9ab8e917f2d7870eaec1c4bcbc963c5f3a52b8045250f62e9e21a091665e67e3d3ede895755b6d98d9aa4e6783c5f99a7c2e54d3d08392a260a10500cb76bb1cf8942363c945b7008baaa026e138a7036df08d9658699576eb3c47ac64c60246685210f0d80b6ea42325eaf9a9727cf61c6d865980e0b686eb45caf6f96c638dad3b160d874cbbd341c39d9c5b4c484f908649cb5e3b7e06b48f8d5d27ff7021d218df9a1be44f23d9e8e13937d33e27563&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:15:37.429425408+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/efaj8qxg/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T071519Z&X-Goog-Expires=86399&X-Goog-Signature=88d460d973e6f91da1a5744f00dec3b92c44067bb213951e7663b743587b52235a7e5f404473dda9615fdd1c8c17d459f36fda57d92ad6d3fad1400dbbcedd45729b477d38605d72354bd45bc31c74923a294114d9c80d818cb864db820f1dc188b9ad3eca4b38e243d004e395a907a874430d1b198fbb63cbdf860803c3bbc91cdaac28bc276bc115603a7a46347c7209bee61f5973744af3abadd4a25e49b513e10b15daf485b9585cfdd9f0822b487ed962a76840936c0d0efea22bca250ea956116c19515bc6c696cb1b7b0df2108335785e9416ebe0f652036f1e7e852633ef7296d65a0d88cf18dbbd22e505bff357337f44fe8d1f57cd226ea45c2d1f&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:52096->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/efaj8qxg/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T071519Z&X-Goog-Expires=86399&X-Goog-Signature=88d460d973e6f91da1a5744f00dec3b92c44067bb213951e7663b743587b52235a7e5f404473dda9615fdd1c8c17d459f36fda57d92ad6d3fad1400dbbcedd45729b477d38605d72354bd45bc31c74923a294114d9c80d818cb864db820f1dc188b9ad3eca4b38e243d004e395a907a874430d1b198fbb63cbdf860803c3bbc91cdaac28bc276bc115603a7a46347c7209bee61f5973744af3abadd4a25e49b513e10b15daf485b9585cfdd9f0822b487ed962a76840936c0d0efea22bca250ea956116c19515bc6c696cb1b7b0df2108335785e9416ebe0f652036f1e7e852633ef7296d65a0d88cf18dbbd22e505bff357337f44fe8d1f57cd226ea45c2d1f&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:15:53.995391811+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/efaj8qxg/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T071519Z&X-Goog-Expires=86399&X-Goog-Signature=88d460d973e6f91da1a5744f00dec3b92c44067bb213951e7663b743587b52235a7e5f404473dda9615fdd1c8c17d459f36fda57d92ad6d3fad1400dbbcedd45729b477d38605d72354bd45bc31c74923a294114d9c80d818cb864db820f1dc188b9ad3eca4b38e243d004e395a907a874430d1b198fbb63cbdf860803c3bbc91cdaac28bc276bc115603a7a46347c7209bee61f5973744af3abadd4a25e49b513e10b15daf485b9585cfdd9f0822b487ed962a76840936c0d0efea22bca250ea956116c19515bc6c696cb1b7b0df2108335785e9416ebe0f652036f1e7e852633ef7296d65a0d88cf18dbbd22e505bff357337f44fe8d1f57cd226ea45c2d1f&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:38546->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/efaj8qxg/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T071519Z&X-Goog-Expires=86399&X-Goog-Signature=88d460d973e6f91da1a5744f00dec3b92c44067bb213951e7663b743587b52235a7e5f404473dda9615fdd1c8c17d459f36fda57d92ad6d3fad1400dbbcedd45729b477d38605d72354bd45bc31c74923a294114d9c80d818cb864db820f1dc188b9ad3eca4b38e243d004e395a907a874430d1b198fbb63cbdf860803c3bbc91cdaac28bc276bc115603a7a46347c7209bee61f5973744af3abadd4a25e49b513e10b15daf485b9585cfdd9f0822b487ed962a76840936c0d0efea22bca250ea956116c19515bc6c696cb1b7b0df2108335785e9416ebe0f652036f1e7e852633ef7296d65a0d88cf18dbbd22e505bff357337f44fe8d1f57cd226ea45c2d1f&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:17:44.11612881+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/efaj8qxg/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T071533Z&X-Goog-Expires=86399&X-Goog-Signature=1628d26f96b0b33d3e020174540165601605cb8a579883e724bdc13410032e4d7278549b76c7506ffe106fa267e0a6d71c7e39a28d191b9cfa5325972ede3828a23d4bc5af149b69ef6af139d71714e0180ad931f5caba09fa31d0e49b819e02ebf51767909e3807e5a43d6ee32224eef9fa78cabd4aa56ae2bd8b2df31768a0d15a44bd9c1bdf03b616a9f7a2f9a2399995c1b2139210a12f3b2347d31fd0fa9f61891bab2c6d013b4a28660880e52028eaa1ebac824bd1fd45e5461e532a398b35e14fdd5e0c861ed3cda677a1e79dcab0c88f16557c7c23922a096350023e95f96cf1214fd3d1023790016c4ee7de72921d4682263acc229c455ca684631e&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:42928->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/efaj8qxg/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T071533Z&X-Goog-Expires=86399&X-Goog-Signature=1628d26f96b0b33d3e020174540165601605cb8a579883e724bdc13410032e4d7278549b76c7506ffe106fa267e0a6d71c7e39a28d191b9cfa5325972ede3828a23d4bc5af149b69ef6af139d71714e0180ad931f5caba09fa31d0e49b819e02ebf51767909e3807e5a43d6ee32224eef9fa78cabd4aa56ae2bd8b2df31768a0d15a44bd9c1bdf03b616a9f7a2f9a2399995c1b2139210a12f3b2347d31fd0fa9f61891bab2c6d013b4a28660880e52028eaa1ebac824bd1fd45e5461e532a398b35e14fdd5e0c861ed3cda677a1e79dcab0c88f16557c7c23922a096350023e95f96cf1214fd3d1023790016c4ee7de72921d4682263acc229c455ca684631e&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:18:41.474642178+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/efaj8qxg/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T071519Z&X-Goog-Expires=86399&X-Goog-Signature=88d460d973e6f91da1a5744f00dec3b92c44067bb213951e7663b743587b52235a7e5f404473dda9615fdd1c8c17d459f36fda57d92ad6d3fad1400dbbcedd45729b477d38605d72354bd45bc31c74923a294114d9c80d818cb864db820f1dc188b9ad3eca4b38e243d004e395a907a874430d1b198fbb63cbdf860803c3bbc91cdaac28bc276bc115603a7a46347c7209bee61f5973744af3abadd4a25e49b513e10b15daf485b9585cfdd9f0822b487ed962a76840936c0d0efea22bca250ea956116c19515bc6c696cb1b7b0df2108335785e9416ebe0f652036f1e7e852633ef7296d65a0d88cf18dbbd22e505bff357337f44fe8d1f57cd226ea45c2d1f&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:43220->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/efaj8qxg/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T071519Z&X-Goog-Expires=86399&X-Goog-Signature=88d460d973e6f91da1a5744f00dec3b92c44067bb213951e7663b743587b52235a7e5f404473dda9615fdd1c8c17d459f36fda57d92ad6d3fad1400dbbcedd45729b477d38605d72354bd45bc31c74923a294114d9c80d818cb864db820f1dc188b9ad3eca4b38e243d004e395a907a874430d1b198fbb63cbdf860803c3bbc91cdaac28bc276bc115603a7a46347c7209bee61f5973744af3abadd4a25e49b513e10b15daf485b9585cfdd9f0822b487ed962a76840936c0d0efea22bca250ea956116c19515bc6c696cb1b7b0df2108335785e9416ebe0f652036f1e7e852633ef7296d65a0d88cf18dbbd22e505bff357337f44fe8d1f57cd226ea45c2d1f&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:19:41.850565691+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/efaj8qxg/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T071519Z&X-Goog-Expires=86399&X-Goog-Signature=88d460d973e6f91da1a5744f00dec3b92c44067bb213951e7663b743587b52235a7e5f404473dda9615fdd1c8c17d459f36fda57d92ad6d3fad1400dbbcedd45729b477d38605d72354bd45bc31c74923a294114d9c80d818cb864db820f1dc188b9ad3eca4b38e243d004e395a907a874430d1b198fbb63cbdf860803c3bbc91cdaac28bc276bc115603a7a46347c7209bee61f5973744af3abadd4a25e49b513e10b15daf485b9585cfdd9f0822b487ed962a76840936c0d0efea22bca250ea956116c19515bc6c696cb1b7b0df2108335785e9416ebe0f652036f1e7e852633ef7296d65a0d88cf18dbbd22e505bff357337f44fe8d1f57cd226ea45c2d1f&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:44610->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/efaj8qxg/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T071519Z&X-Goog-Expires=86399&X-Goog-Signature=88d460d973e6f91da1a5744f00dec3b92c44067bb213951e7663b743587b52235a7e5f404473dda9615fdd1c8c17d459f36fda57d92ad6d3fad1400dbbcedd45729b477d38605d72354bd45bc31c74923a294114d9c80d818cb864db820f1dc188b9ad3eca4b38e243d004e395a907a874430d1b198fbb63cbdf860803c3bbc91cdaac28bc276bc115603a7a46347c7209bee61f5973744af3abadd4a25e49b513e10b15daf485b9585cfdd9f0822b487ed962a76840936c0d0efea22bca250ea956116c19515bc6c696cb1b7b0df2108335785e9416ebe0f652036f1e7e852633ef7296d65a0d88cf18dbbd22e505bff357337f44fe8d1f57cd226ea45c2d1f&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:19:51.078023234+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/efaj8qxg/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T071519Z&X-Goog-Expires=86399&X-Goog-Signature=77515741d3d6a1aa2328f782beb0f48c5167fab6c1266cc5c3a77927d9a89f957b020839fd39de588a5b6e79475a413b53706b3a9dab3039b39000be838c59e1f12fdaaf62bbfbe3fe6698a94a94ed82ca2a461185e513aced2840a000ae77be7140d01a0c944aa1c479631d4195efd42fa935568da32d1a59557bcbd2bffa388928d3811fd59421e339bdff37df9c6075922c4f1e9e24a5a137869eae246fb8658dce98d7b0d331ec6276e324eb033931eadb77ac6517941430d91bdb92b9e27ce3035e5b48f8ca176af529342c13a979efc57dfdd6cfa80e13a5049367a476f0c134de70039d3211b70746a04f0626babfa66d2ec8cfe976ffbd6d91d9d006&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:58550->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/efaj8qxg/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T071519Z&X-Goog-Expires=86399&X-Goog-Signature=77515741d3d6a1aa2328f782beb0f48c5167fab6c1266cc5c3a77927d9a89f957b020839fd39de588a5b6e79475a413b53706b3a9dab3039b39000be838c59e1f12fdaaf62bbfbe3fe6698a94a94ed82ca2a461185e513aced2840a000ae77be7140d01a0c944aa1c479631d4195efd42fa935568da32d1a59557bcbd2bffa388928d3811fd59421e339bdff37df9c6075922c4f1e9e24a5a137869eae246fb8658dce98d7b0d331ec6276e324eb033931eadb77ac6517941430d91bdb92b9e27ce3035e5b48f8ca176af529342c13a979efc57dfdd6cfa80e13a5049367a476f0c134de70039d3211b70746a04f0626babfa66d2ec8cfe976ffbd6d91d9d006&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:19:56.017364221+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/efaj8qxg/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T071519Z&X-Goog-Expires=86399&X-Goog-Signature=77515741d3d6a1aa2328f782beb0f48c5167fab6c1266cc5c3a77927d9a89f957b020839fd39de588a5b6e79475a413b53706b3a9dab3039b39000be838c59e1f12fdaaf62bbfbe3fe6698a94a94ed82ca2a461185e513aced2840a000ae77be7140d01a0c944aa1c479631d4195efd42fa935568da32d1a59557bcbd2bffa388928d3811fd59421e339bdff37df9c6075922c4f1e9e24a5a137869eae246fb8658dce98d7b0d331ec6276e324eb033931eadb77ac6517941430d91bdb92b9e27ce3035e5b48f8ca176af529342c13a979efc57dfdd6cfa80e13a5049367a476f0c134de70039d3211b70746a04f0626babfa66d2ec8cfe976ffbd6d91d9d006&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:37100->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/efaj8qxg/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T071519Z&X-Goog-Expires=86399&X-Goog-Signature=77515741d3d6a1aa2328f782beb0f48c5167fab6c1266cc5c3a77927d9a89f957b020839fd39de588a5b6e79475a413b53706b3a9dab3039b39000be838c59e1f12fdaaf62bbfbe3fe6698a94a94ed82ca2a461185e513aced2840a000ae77be7140d01a0c944aa1c479631d4195efd42fa935568da32d1a59557bcbd2bffa388928d3811fd59421e339bdff37df9c6075922c4f1e9e24a5a137869eae246fb8658dce98d7b0d331ec6276e324eb033931eadb77ac6517941430d91bdb92b9e27ce3035e5b48f8ca176af529342c13a979efc57dfdd6cfa80e13a5049367a476f0c134de70039d3211b70746a04f0626babfa66d2ec8cfe976ffbd6d91d9d006&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:19:59.27231191+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/efaj8qxg/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T071533Z&X-Goog-Expires=86399&X-Goog-Signature=728b16e5ae45a557d355aca5d47a1f22a749044e5f0894f502ef5cb272aaa29f6019d3c2a2f8b2740ab94f241db698c1d657d871c1d6bb4cb25559fcc4e00da68a9fb13eeb707c463237c86932d3a79fef4d3edffdf8131181058584c6020341ae70ea30517affe9cd275076f7d88080e1dec3a9e3aa0ce70f6106272707976fe828f31a585b11218b205b2ea368c257d9dcf6da21893a76a1faf06c772f07df6d06aa1677a1d8c0823cd05b0054356c8b4daa4d575ed6faaca834c60a6f4b07b21b9e8ae07bca84f90ffd581a95209aab1f7b52d84f096f1348383e9fd7871890421f633299ece7e32cc66d05c8f171a7ecccf16756f013b97c35e64bceffe0&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:57352->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/efaj8qxg/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T071533Z&X-Goog-Expires=86399&X-Goog-Signature=728b16e5ae45a557d355aca5d47a1f22a749044e5f0894f502ef5cb272aaa29f6019d3c2a2f8b2740ab94f241db698c1d657d871c1d6bb4cb25559fcc4e00da68a9fb13eeb707c463237c86932d3a79fef4d3edffdf8131181058584c6020341ae70ea30517affe9cd275076f7d88080e1dec3a9e3aa0ce70f6106272707976fe828f31a585b11218b205b2ea368c257d9dcf6da21893a76a1faf06c772f07df6d06aa1677a1d8c0823cd05b0054356c8b4daa4d575ed6faaca834c60a6f4b07b21b9e8ae07bca84f90ffd581a95209aab1f7b52d84f096f1348383e9fd7871890421f633299ece7e32cc66d05c8f171a7ecccf16756f013b97c35e64bceffe0&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:20:03.486003743+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/efaj8qxg/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T071533Z&X-Goog-Expires=86399&X-Goog-Signature=9a05e89c850dc41833db428d1fdf4d303a95501e1a60518ddbb1761d0f33be57bb26a9c84ffbe52221822fa259f3a5f21e797ce8c8783413cb8c14f3c8cb86a0acfa17940b8844ad19bb8031ff76c9ab8e917f2d7870eaec1c4bcbc963c5f3a52b8045250f62e9e21a091665e67e3d3ede895755b6d98d9aa4e6783c5f99a7c2e54d3d08392a260a10500cb76bb1cf8942363c945b7008baaa026e138a7036df08d9658699576eb3c47ac64c60246685210f0d80b6ea42325eaf9a9727cf61c6d865980e0b686eb45caf6f96c638dad3b160d874cbbd341c39d9c5b4c484f908649cb5e3b7e06b48f8d5d27ff7021d218df9a1be44f23d9e8e13937d33e27563&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:49850->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/efaj8qxg/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T071533Z&X-Goog-Expires=86399&X-Goog-Signature=9a05e89c850dc41833db428d1fdf4d303a95501e1a60518ddbb1761d0f33be57bb26a9c84ffbe52221822fa259f3a5f21e797ce8c8783413cb8c14f3c8cb86a0acfa17940b8844ad19bb8031ff76c9ab8e917f2d7870eaec1c4bcbc963c5f3a52b8045250f62e9e21a091665e67e3d3ede895755b6d98d9aa4e6783c5f99a7c2e54d3d08392a260a10500cb76bb1cf8942363c945b7008baaa026e138a7036df08d9658699576eb3c47ac64c60246685210f0d80b6ea42325eaf9a9727cf61c6d865980e0b686eb45caf6f96c638dad3b160d874cbbd341c39d9c5b4c484f908649cb5e3b7e06b48f8d5d27ff7021d218df9a1be44f23d9e8e13937d33e27563&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:20:05.341606002+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/efaj8qxg/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T071519Z&X-Goog-Expires=86399&X-Goog-Signature=77515741d3d6a1aa2328f782beb0f48c5167fab6c1266cc5c3a77927d9a89f957b020839fd39de588a5b6e79475a413b53706b3a9dab3039b39000be838c59e1f12fdaaf62bbfbe3fe6698a94a94ed82ca2a461185e513aced2840a000ae77be7140d01a0c944aa1c479631d4195efd42fa935568da32d1a59557bcbd2bffa388928d3811fd59421e339bdff37df9c6075922c4f1e9e24a5a137869eae246fb8658dce98d7b0d331ec6276e324eb033931eadb77ac6517941430d91bdb92b9e27ce3035e5b48f8ca176af529342c13a979efc57dfdd6cfa80e13a5049367a476f0c134de70039d3211b70746a04f0626babfa66d2ec8cfe976ffbd6d91d9d006&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:59880->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/efaj8qxg/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T071519Z&X-Goog-Expires=86399&X-Goog-Signature=77515741d3d6a1aa2328f782beb0f48c5167fab6c1266cc5c3a77927d9a89f957b020839fd39de588a5b6e79475a413b53706b3a9dab3039b39000be838c59e1f12fdaaf62bbfbe3fe6698a94a94ed82ca2a461185e513aced2840a000ae77be7140d01a0c944aa1c479631d4195efd42fa935568da32d1a59557bcbd2bffa388928d3811fd59421e339bdff37df9c6075922c4f1e9e24a5a137869eae246fb8658dce98d7b0d331ec6276e324eb033931eadb77ac6517941430d91bdb92b9e27ce3035e5b48f8ca176af529342c13a979efc57dfdd6cfa80e13a5049367a476f0c134de70039d3211b70746a04f0626babfa66d2ec8cfe976ffbd6d91d9d006&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:20:12.525935837+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/efaj8qxg/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T071533Z&X-Goog-Expires=86399&X-Goog-Signature=9a05e89c850dc41833db428d1fdf4d303a95501e1a60518ddbb1761d0f33be57bb26a9c84ffbe52221822fa259f3a5f21e797ce8c8783413cb8c14f3c8cb86a0acfa17940b8844ad19bb8031ff76c9ab8e917f2d7870eaec1c4bcbc963c5f3a52b8045250f62e9e21a091665e67e3d3ede895755b6d98d9aa4e6783c5f99a7c2e54d3d08392a260a10500cb76bb1cf8942363c945b7008baaa026e138a7036df08d9658699576eb3c47ac64c60246685210f0d80b6ea42325eaf9a9727cf61c6d865980e0b686eb45caf6f96c638dad3b160d874cbbd341c39d9c5b4c484f908649cb5e3b7e06b48f8d5d27ff7021d218df9a1be44f23d9e8e13937d33e27563&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:58412->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/efaj8qxg/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T071533Z&X-Goog-Expires=86399&X-Goog-Signature=9a05e89c850dc41833db428d1fdf4d303a95501e1a60518ddbb1761d0f33be57bb26a9c84ffbe52221822fa259f3a5f21e797ce8c8783413cb8c14f3c8cb86a0acfa17940b8844ad19bb8031ff76c9ab8e917f2d7870eaec1c4bcbc963c5f3a52b8045250f62e9e21a091665e67e3d3ede895755b6d98d9aa4e6783c5f99a7c2e54d3d08392a260a10500cb76bb1cf8942363c945b7008baaa026e138a7036df08d9658699576eb3c47ac64c60246685210f0d80b6ea42325eaf9a9727cf61c6d865980e0b686eb45caf6f96c638dad3b160d874cbbd341c39d9c5b4c484f908649cb5e3b7e06b48f8d5d27ff7021d218df9a1be44f23d9e8e13937d33e27563&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:20:31.650287026+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/efaj8qxg/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T071533Z&X-Goog-Expires=86399&X-Goog-Signature=9a05e89c850dc41833db428d1fdf4d303a95501e1a60518ddbb1761d0f33be57bb26a9c84ffbe52221822fa259f3a5f21e797ce8c8783413cb8c14f3c8cb86a0acfa17940b8844ad19bb8031ff76c9ab8e917f2d7870eaec1c4bcbc963c5f3a52b8045250f62e9e21a091665e67e3d3ede895755b6d98d9aa4e6783c5f99a7c2e54d3d08392a260a10500cb76bb1cf8942363c945b7008baaa026e138a7036df08d9658699576eb3c47ac64c60246685210f0d80b6ea42325eaf9a9727cf61c6d865980e0b686eb45caf6f96c638dad3b160d874cbbd341c39d9c5b4c484f908649cb5e3b7e06b48f8d5d27ff7021d218df9a1be44f23d9e8e13937d33e27563&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:33484->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/efaj8qxg/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T071533Z&X-Goog-Expires=86399&X-Goog-Signature=9a05e89c850dc41833db428d1fdf4d303a95501e1a60518ddbb1761d0f33be57bb26a9c84ffbe52221822fa259f3a5f21e797ce8c8783413cb8c14f3c8cb86a0acfa17940b8844ad19bb8031ff76c9ab8e917f2d7870eaec1c4bcbc963c5f3a52b8045250f62e9e21a091665e67e3d3ede895755b6d98d9aa4e6783c5f99a7c2e54d3d08392a260a10500cb76bb1cf8942363c945b7008baaa026e138a7036df08d9658699576eb3c47ac64c60246685210f0d80b6ea42325eaf9a9727cf61c6d865980e0b686eb45caf6f96c638dad3b160d874cbbd341c39d9c5b4c484f908649cb5e3b7e06b48f8d5d27ff7021d218df9a1be44f23d9e8e13937d33e27563&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:21:08.782300918+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/efaj8qxg/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T071533Z&X-Goog-Expires=86399&X-Goog-Signature=9a05e89c850dc41833db428d1fdf4d303a95501e1a60518ddbb1761d0f33be57bb26a9c84ffbe52221822fa259f3a5f21e797ce8c8783413cb8c14f3c8cb86a0acfa17940b8844ad19bb8031ff76c9ab8e917f2d7870eaec1c4bcbc963c5f3a52b8045250f62e9e21a091665e67e3d3ede895755b6d98d9aa4e6783c5f99a7c2e54d3d08392a260a10500cb76bb1cf8942363c945b7008baaa026e138a7036df08d9658699576eb3c47ac64c60246685210f0d80b6ea42325eaf9a9727cf61c6d865980e0b686eb45caf6f96c638dad3b160d874cbbd341c39d9c5b4c484f908649cb5e3b7e06b48f8d5d27ff7021d218df9a1be44f23d9e8e13937d33e27563&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:45056->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/efaj8qxg/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T071533Z&X-Goog-Expires=86399&X-Goog-Signature=9a05e89c850dc41833db428d1fdf4d303a95501e1a60518ddbb1761d0f33be57bb26a9c84ffbe52221822fa259f3a5f21e797ce8c8783413cb8c14f3c8cb86a0acfa17940b8844ad19bb8031ff76c9ab8e917f2d7870eaec1c4bcbc963c5f3a52b8045250f62e9e21a091665e67e3d3ede895755b6d98d9aa4e6783c5f99a7c2e54d3d08392a260a10500cb76bb1cf8942363c945b7008baaa026e138a7036df08d9658699576eb3c47ac64c60246685210f0d80b6ea42325eaf9a9727cf61c6d865980e0b686eb45caf6f96c638dad3b160d874cbbd341c39d9c5b4c484f908649cb5e3b7e06b48f8d5d27ff7021d218df9a1be44f23d9e8e13937d33e27563&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:22:09.118833793+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-08-13T15:22:12.928205537+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/protein-llm-contrastive/efaj8qxg/file_stream\": read tcp 10.1.1.90:46630->104.21.20.172:443: read: connection reset by peer"} +{"time":"2025-08-13T15:22:26.373143026+08:00","level":"INFO","msg":"handler: closed","stream_id":"efaj8qxg"} +{"time":"2025-08-13T15:22:26.377067528+08:00","level":"INFO","msg":"sender: closed","stream_id":"efaj8qxg"} +{"time":"2025-08-13T15:22:26.377100954+08:00","level":"INFO","msg":"stream: closed","id":"efaj8qxg"} diff --git a/BioReason_new/wandb/run-20250813_151502-efaj8qxg/logs/debug.log b/BioReason_new/wandb/run-20250813_151502-efaj8qxg/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..c3343c9f3f09f202149c1cc36dbc5323b4a6486e --- /dev/null +++ b/BioReason_new/wandb/run-20250813_151502-efaj8qxg/logs/debug.log @@ -0,0 +1,23 @@ +2025-08-13 15:15:02,666 INFO MainThread:11722 [wandb_setup.py:_flush():80] Current SDK version is 0.21.1 +2025-08-13 15:15:02,666 INFO MainThread:11722 [wandb_setup.py:_flush():80] Configure stats pid to 11722 +2025-08-13 15:15:02,666 INFO MainThread:11722 [wandb_setup.py:_flush():80] Loading settings from /root/.config/wandb/settings +2025-08-13 15:15:02,666 INFO MainThread:11722 [wandb_setup.py:_flush():80] Loading settings from /nas/shared/kilab/wangyujia/BioReason_new/wandb/settings +2025-08-13 15:15:02,666 INFO MainThread:11722 [wandb_setup.py:_flush():80] Loading settings from environment variables +2025-08-13 15:15:02,666 INFO MainThread:11722 [wandb_init.py:setup_run_log_directory():703] Logging user logs to /nas/shared/kilab/wangyujia/BioReason_new/wandb/run-20250813_151502-efaj8qxg/logs/debug.log +2025-08-13 15:15:02,666 INFO MainThread:11722 [wandb_init.py:setup_run_log_directory():704] Logging internal logs to /nas/shared/kilab/wangyujia/BioReason_new/wandb/run-20250813_151502-efaj8qxg/logs/debug-internal.log +2025-08-13 15:15:02,666 INFO MainThread:11722 [wandb_init.py:init():830] calling init triggers +2025-08-13 15:15:02,666 INFO MainThread:11722 [wandb_init.py:init():835] wandb.init called with sweep_config: {} +config: {'text_model_name': '/oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged', 'protein_model_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m', 'qformer_model_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft', 'cache_dir': '/model-weights', 'num_query_tokens': 8, 'train_dataset': '/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl', 'valid_dataset': '/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl', 'eval_dataset': True, 'output_dir': './contrastive_outputs', 'num_epochs': 10, 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 0.01, 'warmup_steps': 1000, 'gradient_accumulation_steps': 1, 'temperature': 0.07, 'freeze_protein_model': True, 'freeze_text_model': True, 'protein_weight': 1.0, 'text_weight': 1.0, 'enable_ptm': True, 'ptm_weight': 1.0, 'max_length_protein': 1024, 'max_length_text': 512, 'num_workers': 8, 'logging_steps': 100, 'eval_steps': 500, 'save_steps': 1000, 'save_total_limit': 3, 'fp16': False, 'bf16': False, 'seed': 42, 'use_wandb': True, 'wandb_project': 'protein-llm-contrastive', 'wandb_entity': None, '_wandb': {}} +2025-08-13 15:15:02,666 INFO MainThread:11722 [wandb_init.py:init():871] starting backend +2025-08-13 15:15:02,875 INFO MainThread:11722 [wandb_init.py:init():874] sending inform_init request +2025-08-13 15:15:02,882 INFO MainThread:11722 [wandb_init.py:init():882] backend started and connected +2025-08-13 15:15:02,883 INFO MainThread:11722 [wandb_init.py:init():953] updated telemetry +2025-08-13 15:15:02,925 INFO MainThread:11722 [wandb_init.py:init():977] communicating run to backend with 90.0 second timeout +2025-08-13 15:15:19,097 INFO MainThread:11722 [wandb_init.py:init():1029] starting run threads in backend +2025-08-13 15:15:19,205 INFO MainThread:11722 [wandb_run.py:_console_start():2494] atexit reg +2025-08-13 15:15:19,206 INFO MainThread:11722 [wandb_run.py:_redirect():2342] redirect: wrap_raw +2025-08-13 15:15:19,206 INFO MainThread:11722 [wandb_run.py:_redirect():2411] Wrapping output streams. +2025-08-13 15:15:19,206 INFO MainThread:11722 [wandb_run.py:_redirect():2434] Redirects installed. +2025-08-13 15:15:19,208 INFO MainThread:11722 [wandb_init.py:init():1075] run started, returning control to user process +2025-08-13 15:15:28,936 INFO MainThread:11722 [wandb_run.py:_config_callback():1380] config_cb None None {'output_dir': './contrastive_outputs', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': True, 'do_predict': False, 'eval_strategy': 'steps', 'prediction_loss_only': False, 'per_device_train_batch_size': 8, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'eval_delay': 0, 'torch_empty_cache_steps': None, 'learning_rate': 0.0001, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 10, 'max_steps': -1, 'lr_scheduler_type': 'linear', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.0, 'warmup_steps': 1000, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': './contrastive_outputs/runs/Aug13_15-15-23_dsw-265304-585cc9d768-ckfd7', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 100, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 1000, 'save_total_limit': 3, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'restore_callback_states_from_checkpoint': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 42, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': False, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 500, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': None, 'disable_tqdm': False, 'remove_unused_columns': False, 'label_names': None, 'load_best_model_at_end': True, 'metric_for_best_model': 'eval_avg_recall_at_1', 'greater_is_better': True, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'deepspeed': None, 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['wandb'], 'ddp_find_unused_parameters': False, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '', 'hub_private_repo': None, 'hub_always_push': False, 'hub_revision': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'include_for_metrics': [], 'eval_do_concat_batches': True, 'fp16_backend': 'auto', 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None, 'batch_eval_metrics': False, 'eval_on_start': False, 'use_liger_kernel': False, 'liger_kernel_config': None, 'eval_use_gather_object': False, 'average_tokens_across_devices': False, 'temperature': 0.07, 'freeze_protein_model': True, 'freeze_text_model': True, 'protein_weight': 1.0, 'text_weight': 1.0, 'max_length_protein': 1024, 'max_length_text': 512, 'enable_ptm': True, 'ptm_weight': 1.0} +2025-08-13 15:15:33,212 INFO MsgRouterThr:11722 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles. diff --git a/BioReason_new/wandb/run-20250813_151502-efaj8qxg/run-efaj8qxg.wandb b/BioReason_new/wandb/run-20250813_151502-efaj8qxg/run-efaj8qxg.wandb new file mode 100644 index 0000000000000000000000000000000000000000..5c5c332d5ccb72e11310c5ebe5e979a731db9ff1 Binary files /dev/null and b/BioReason_new/wandb/run-20250813_151502-efaj8qxg/run-efaj8qxg.wandb differ diff --git a/BioReason_new/wandb/run-20250813_153318-ig4rhoqf/files/config.yaml b/BioReason_new/wandb/run-20250813_153318-ig4rhoqf/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1a5e602d026ba8198435fd6f44e5c31a7c869d40 --- /dev/null +++ b/BioReason_new/wandb/run-20250813_153318-ig4rhoqf/files/config.yaml @@ -0,0 +1,457 @@ +_wandb: + value: + cli_version: 0.21.1 + e: + 6fva4lshb0nkqoae8k41gurhtr9m5gpr: + args: + - --text_model_name + - /oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged + - --protein_model_name + - /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m + - --qformer_model_name + - /nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft + - --num_query_tokens + - "8" + - --train_dataset + - /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl + - --valid_dataset + - /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl + - --output_dir + - ./contrastive_outputs + - --num_epochs + - "10" + - --batch_size + - "8" + - --learning_rate + - "1e-4" + - --temperature + - "0.07" + - --freeze_protein_model + - --freeze_text_model + - --enable_ptm + - --max_length_protein + - "1024" + - --max_length_text + - "512" + - --num_workers + - "8" + - --eval_dataset + - --use_wandb + - --wandb_project + - protein-llm-contrastive + - --logging_steps + - "100" + - --eval_steps + - "500" + - --save_steps + - "1000" + codePath: wangyujia/BioReason_new/train_contrastive.py + codePathLocal: train_contrastive.py + cpu_count: 64 + cpu_count_logical: 64 + cudaVersion: "12.1" + disk: + /: + total: "1623302262784" + used: "1015304192" + executable: /root/miniconda3/envs/bioreason/bin/python + git: + commit: b8caf406aa1699c788f0ca6e44a1769452c317db + remote: https://github.com/PorUna-byte/PAR.git + gpu: NVIDIA A800-SXM4-80GB + gpu_count: 8 + gpu_nvidia: + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-62fcc532-880f-7324-eb13-3b739a38c29b + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-af1197aa-3e7d-a53e-185d-04b1d64164da + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-3e6e551f-2d7d-a27b-654a-da56933d2283 + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-a947b64a-c545-ee5e-30e6-b4b0e2dd8fdc + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-ebd04f3f-c437-1b8b-26f2-4b1ebfee93c4 + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-53994891-72a6-25be-a4c8-e32c695155b0 + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-a927d00a-15c5-31f5-9ae1-9d2e773821a1 + - architecture: Ampere + name: NVIDIA A800-SXM4-80GB + uuid: GPU-93d555c4-29a0-cd78-abac-d98cd160d4e1 + host: dsw-265304-585cc9d768-ckfd7 + memory: + total: "549755813888" + os: Linux-5.10.134-008.18.kangaroo.al8.x86_64-x86_64-with-glibc2.35 + program: /nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py + python: CPython 3.11.0 + root: /nas/shared/kilab/wangyujia/BioReason_new + startedAt: "2025-08-13T07:33:18.645783Z" + writerId: 6fva4lshb0nkqoae8k41gurhtr9m5gpr + m: + - "1": train/global_step + "6": + - 3 + "7": [] + - "2": '*' + "5": 1 + "6": + - 1 + "7": [] + python_version: 3.11.0 + t: + "1": + - 1 + - 9 + - 11 + - 41 + - 49 + - 51 + - 71 + - 84 + - 98 + - 103 + "2": + - 1 + - 9 + - 11 + - 41 + - 49 + - 51 + - 71 + - 84 + - 98 + - 103 + "3": + - 7 + - 13 + - 16 + - 66 + "4": 3.11.0 + "5": 0.21.1 + "6": 4.55.0 + "9": + "1": transformers_trainer + "12": 0.21.1 + "13": linux-x86_64 +accelerator_config: + value: + dispatch_batches: null + even_batches: true + gradient_accumulation_kwargs: null + non_blocking: false + split_batches: false + use_seedable_sampler: true +adafactor: + value: false +adam_beta1: + value: 0.9 +adam_beta2: + value: 0.999 +adam_epsilon: + value: 1e-08 +auto_find_batch_size: + value: false +average_tokens_across_devices: + value: false +batch_eval_metrics: + value: false +batch_size: + value: 8 +bf16: + value: false +bf16_full_eval: + value: false +cache_dir: + value: /model-weights +data_seed: + value: null +dataloader_drop_last: + value: false +dataloader_num_workers: + value: 8 +dataloader_persistent_workers: + value: false +dataloader_pin_memory: + value: true +dataloader_prefetch_factor: + value: null +ddp_backend: + value: null +ddp_broadcast_buffers: + value: null +ddp_bucket_cap_mb: + value: null +ddp_find_unused_parameters: + value: false +ddp_timeout: + value: 1800 +debug: + value: [] +deepspeed: + value: null +disable_tqdm: + value: false +do_eval: + value: true +do_predict: + value: false +do_train: + value: false +enable_ptm: + value: true +eval_accumulation_steps: + value: null +eval_dataset: + value: true +eval_delay: + value: 0 +eval_do_concat_batches: + value: true +eval_on_start: + value: false +eval_steps: + value: 500 +eval_strategy: + value: steps +eval_use_gather_object: + value: false +fp16: + value: false +fp16_backend: + value: auto +fp16_full_eval: + value: false +fp16_opt_level: + value: O1 +freeze_protein_model: + value: true +freeze_text_model: + value: true +fsdp: + value: [] +fsdp_config: + value: + min_num_params: 0 + xla: false + xla_fsdp_grad_ckpt: false + xla_fsdp_v2: false +fsdp_min_num_params: + value: 0 +fsdp_transformer_layer_cls_to_wrap: + value: null +full_determinism: + value: false +gradient_accumulation_steps: + value: 1 +gradient_checkpointing: + value: false +gradient_checkpointing_kwargs: + value: null +greater_is_better: + value: true +group_by_length: + value: false +half_precision_backend: + value: auto +hub_always_push: + value: false +hub_model_id: + value: null +hub_private_repo: + value: null +hub_revision: + value: null +hub_strategy: + value: every_save +hub_token: + value: +ignore_data_skip: + value: false +include_for_metrics: + value: [] +include_inputs_for_metrics: + value: false +include_num_input_tokens_seen: + value: false +include_tokens_per_second: + value: false +jit_mode_eval: + value: false +label_names: + value: null +label_smoothing_factor: + value: 0 +learning_rate: + value: 0.0001 +length_column_name: + value: length +liger_kernel_config: + value: null +load_best_model_at_end: + value: true +local_rank: + value: 0 +log_level: + value: passive +log_level_replica: + value: warning +log_on_each_node: + value: true +logging_dir: + value: ./contrastive_outputs/runs/Aug13_15-33-30_dsw-265304-585cc9d768-ckfd7 +logging_first_step: + value: false +logging_nan_inf_filter: + value: true +logging_steps: + value: 100 +logging_strategy: + value: steps +lr_scheduler_type: + value: linear +max_grad_norm: + value: 1 +max_length_protein: + value: 1024 +max_length_text: + value: 512 +max_steps: + value: -1 +metric_for_best_model: + value: eval_avg_recall_at_1 +mp_parameters: + value: "" +neftune_noise_alpha: + value: null +no_cuda: + value: false +num_epochs: + value: 10 +num_query_tokens: + value: 8 +num_train_epochs: + value: 10 +num_workers: + value: 8 +optim: + value: adamw_torch +optim_args: + value: null +optim_target_modules: + value: null +output_dir: + value: ./contrastive_outputs +overwrite_output_dir: + value: false +past_index: + value: -1 +per_device_eval_batch_size: + value: 8 +per_device_train_batch_size: + value: 8 +per_gpu_eval_batch_size: + value: null +per_gpu_train_batch_size: + value: null +prediction_loss_only: + value: false +protein_model_name: + value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m +protein_weight: + value: 1 +ptm_weight: + value: 1 +push_to_hub: + value: false +push_to_hub_model_id: + value: null +push_to_hub_organization: + value: null +push_to_hub_token: + value: +qformer_model_name: + value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft +ray_scope: + value: last +remove_unused_columns: + value: false +report_to: + value: + - wandb +restore_callback_states_from_checkpoint: + value: false +resume_from_checkpoint: + value: null +run_name: + value: null +save_on_each_node: + value: false +save_only_model: + value: false +save_safetensors: + value: true +save_steps: + value: 1000 +save_strategy: + value: steps +save_total_limit: + value: 3 +seed: + value: 42 +skip_memory_metrics: + value: true +temperature: + value: 0.07 +text_model_name: + value: /oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged +text_weight: + value: 1 +tf32: + value: null +torch_compile: + value: false +torch_compile_backend: + value: null +torch_compile_mode: + value: null +torch_empty_cache_steps: + value: null +torchdynamo: + value: null +tpu_metrics_debug: + value: false +tpu_num_cores: + value: null +train_dataset: + value: /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl +use_cpu: + value: false +use_ipex: + value: false +use_legacy_prediction_loop: + value: false +use_liger_kernel: + value: false +use_mps_device: + value: false +use_wandb: + value: true +valid_dataset: + value: /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl +wandb_entity: + value: null +wandb_project: + value: protein-llm-contrastive +warmup_ratio: + value: 0 +warmup_steps: + value: 1000 +weight_decay: + value: 0.01 diff --git a/BioReason_new/wandb/run-20250813_153318-ig4rhoqf/files/output.log b/BioReason_new/wandb/run-20250813_153318-ig4rhoqf/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..2511adb9928da56fab411c97355cab7fcdde75d5 --- /dev/null +++ b/BioReason_new/wandb/run-20250813_153318-ig4rhoqf/files/output.log @@ -0,0 +1,53 @@ +Loading model... +Loading checkpoint shards: 100%|██████████████████████████████████████████████████████| 4/4 [00:01<00:00, 2.44it/s] +Some weights of EsmModel were not initialized from the model checkpoint at /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m and are newly initialized: ['pooler.dense.bias', 'pooler.dense.weight'] +You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference. +Loading datasets... +Training dataset size: 430595 +Eval dataset size: 10000 +Initializing enhanced trainer... +[2025-08-13 15:33:34,781] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) +/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/_distutils_hack/__init__.py:53: UserWarning: Reliance on distutils from stdlib is deprecated. Users must rely on setuptools to provide the distutils module. Avoid importing distutils or import setuptools first, and avoid setting SETUPTOOLS_USE_DISTUTILS=stdlib. Register concerns at https://github.com/pypa/setuptools/issues/new?template=distutils-deprecation.yml + warnings.warn( +[2025-08-13 15:33:35,753] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False +Starting enhanced contrastive training... +- Contrastive learning enabled +- Protein-text matching: enabled +- Temperature: 0.07 +- PTM weight: 1.0 + 0%| | 0/67290 [00:00 + trainer = main(args) + ^^^^^^^^^^ + File "/nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py", line 399, in main + trainer.train() + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/trainer.py", line 2238, in train + return inner_training_loop( + ^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/trainer.py", line 2582, in _inner_training_loop + tr_loss_step = self.training_step(model, inputs, num_items_in_batch) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/trainer.py", line 3796, in training_step + loss = self.compute_loss(model, inputs, num_items_in_batch=num_items_in_batch) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/nas/shared/kilab/wangyujia/BioReason_new/bioreason/trainer/contrast_trainer_new.py", line 351, in compute_loss + loss_ptm = self.contrastive_loss.compute_ptm_loss( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/nas/shared/kilab/wangyujia/BioReason_new/bioreason/trainer/contrast_trainer_new.py", line 211, in compute_ptm_loss + output_ptm = qformer( + ^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1762, in _call_impl + return forward_call(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/models/bert/modeling_bert.py", line 972, in forward + extended_attention_mask = _prepare_4d_attention_mask_for_sdpa( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/miniconda3/envs/bioreason/lib/python3.11/site-packages/transformers/modeling_attn_mask_utils.py", line 454, in _prepare_4d_attention_mask_for_sdpa + if not is_tracing and torch.all(mask == 1): +RuntimeError: CUDA error: device-side assert triggered +CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. +For debugging consider passing CUDA_LAUNCH_BLOCKING=1 +Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions. diff --git a/BioReason_new/wandb/run-20250813_153318-ig4rhoqf/files/requirements.txt b/BioReason_new/wandb/run-20250813_153318-ig4rhoqf/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..743d03fe27e0b7e200935023c07a1d7fa61b99ce --- /dev/null +++ b/BioReason_new/wandb/run-20250813_153318-ig4rhoqf/files/requirements.txt @@ -0,0 +1,233 @@ +jupyterlab_pygments==0.3.0 +depyf==0.19.0 +av==15.0.0 +fastapi==0.116.1 +ray==2.48.0 +cachetools==6.1.0 +prometheus_client==0.22.1 +requests==2.32.4 +jupyter-lsp==2.2.6 +jupyter-events==0.12.0 +anyio==4.10.0 +pybase64==1.4.2 +fastjsonschema==2.21.1 +cloudpickle==3.1.1 +sentry-sdk==2.34.1 +torchmetrics==1.8.1 +wheel==0.45.1 +idna==3.10 +defusedxml==0.7.1 +starlette==0.47.2 +python-dotenv==1.1.1 +comm==0.2.3 +multidict==6.6.3 +python-json-logger==3.3.0 +jupyter_server==2.16.0 +nvidia-nvtx-cu12==12.6.77 +websockets==15.0.1 +referencing==0.36.2 +gguf==0.17.1 +lm-format-enforcer==0.10.12 +xformers==0.0.31 +ninja==1.11.1.4 +beautifulsoup4==4.13.4 +dnspython==2.7.0 +certifi==2025.8.3 +markdown-it-py==3.0.0 +stack-data==0.6.3 +nvidia-cusparselt-cu12==0.6.3 +networkx==3.5 +aiohappyeyeballs==2.6.1 +nvidia-cuda-cupti-cu12==12.6.80 +llguidance==0.7.30 +rich==14.1.0 +outlines_core==0.2.10 +nvidia-nccl-cu12==2.26.2 +pandas==2.3.1 +pydantic_core==2.33.2 +argon2-cffi==25.1.0 +babel==2.17.0 +isoduration==20.11.0 +uvloop==0.21.0 +json5==0.12.0 +widgetsnbextension==4.0.14 +aiohttp==3.12.15 +notebook_shim==0.2.4 +pandocfilters==1.5.1 +vllm==0.10.0 +blake3==1.0.5 +diskcache==5.6.3 +h11==0.16.0 +attrs==25.3.0 +nvidia-cusolver-cu12==11.7.1.2 +click==8.2.1 +cupy-cuda12x==13.5.1 +jedi==0.19.2 +py-cpuinfo==9.0.0 +overrides==7.7.0 +pytz==2025.2 +protobuf==6.31.1 +dill==0.3.8 +ipykernel==6.30.1 +frozenlist==1.7.0 +huggingface-hub==0.34.4 +rignore==0.6.4 +mistune==3.1.3 +sniffio==1.3.1 +httpx==0.28.1 +sentencepiece==0.2.0 +typing_extensions==4.14.1 +typing-inspection==0.4.1 +async-lru==2.0.5 +bioreason==0.1.0 +charset-normalizer==3.4.3 +llvmlite==0.44.0 +safetensors==0.6.2 +fqdn==1.5.1 +peft==0.17.0 +fastapi-cli==0.0.8 +torch==2.7.1 +jsonschema==4.25.0 +torchvision==0.22.1 +parso==0.8.4 +pillow==11.3.0 +pycountry==24.6.1 +scipy==1.16.1 +wandb==0.21.1 +pip==25.1 +wcwidth==0.2.13 +mistral_common==1.8.3 +nvidia-cudnn-cu12==9.5.1.17 +asttokens==3.0.0 +numpy==2.2.6 +uvicorn==0.35.0 +jupyter_client==8.6.3 +debugpy==1.8.16 +fastapi-cloud-cli==0.1.5 +compressed-tensors==0.10.2 +GitPython==3.1.45 +nvidia-ml-py==13.580.65 +tiktoken==0.11.0 +partial-json-parser==0.2.1.1.post6 +prompt_toolkit==3.0.51 +uri-template==1.3.0 +pyarrow==21.0.0 +nvidia-cuda-nvrtc-cu12==12.6.77 +propcache==0.3.2 +matplotlib-inline==0.1.7 +pure_eval==0.2.3 +nest-asyncio==1.6.0 +deepspeed==0.17.4 +einops==0.8.1 +triton==3.3.1 +trl==0.21.0 +pyzmq==27.0.1 +httptools==0.6.4 +ipython==9.4.0 +packaging==25.0 +hjson==3.1.0 +soxr==0.5.0.post1 +webcolors==24.11.1 +PyYAML==6.0.2 +ipywidgets==8.1.7 +lark==1.2.2 +python-multipart==0.0.20 +tinycss2==1.4.0 +mpmath==1.3.0 +psutil==7.0.0 +nbformat==5.10.4 +qwen-vl-utils==0.0.11 +notebook==7.4.5 +jupyterlab==4.4.5 +transformers==4.55.0 +urllib3==2.5.0 +terminado==0.18.1 +xgrammar==0.1.21 +xxhash==3.5.0 +tzdata==2025.2 +jupyterlab_widgets==3.0.15 +aiosignal==1.4.0 +annotated-types==0.7.0 +nvidia-nvjitlink-cu12==12.6.85 +httpcore==1.0.9 +pexpect==4.9.0 +tornado==6.5.2 +regex==2025.7.33 +filelock==3.18.0 +nvidia-cuda-runtime-cu12==12.6.77 +openai==1.90.0 +Pygments==2.19.2 +cffi==1.17.1 +nvidia-cufft-cu12==11.3.0.4 +rpds-py==0.27.0 +rich-toolkit==0.15.0 +bleach==6.2.0 +sympy==1.14.0 +python-dateutil==2.9.0.post0 +watchfiles==1.1.0 +nbclient==0.10.2 +hf-xet==1.1.7 +Send2Trash==1.8.3 +smmap==5.0.2 +tokenizers==0.21.4 +yarl==1.20.1 +MarkupSafe==3.0.2 +prometheus-fastapi-instrumentator==7.1.0 +rfc3986-validator==0.1.1 +bitsandbytes==0.46.1 +nvidia-cufile-cu12==1.11.1.6 +fsspec==2025.3.0 +executing==2.2.0 +pydantic==2.11.7 +jupyter_core==5.8.1 +msgspec==0.19.0 +nvidia-cusparse-cu12==12.5.4.2 +opencv-python-headless==4.12.0.88 +webencodings==0.5.1 +ptyprocess==0.7.0 +pytorch-lightning==2.5.2 +Jinja2==3.1.6 +websocket-client==1.8.0 +jsonpointer==3.0.0 +traitlets==5.14.3 +numba==0.61.2 +soupsieve==2.7 +cbor2==5.6.5 +multiprocess==0.70.16 +email_validator==2.2.0 +distro==1.9.0 +accelerate==1.10.0 +torchaudio==2.7.1 +jiter==0.10.0 +rfc3339-validator==0.1.4 +jupyter==1.1.1 +typer==0.16.0 +gitdb==4.0.12 +types-python-dateutil==2.9.0.20250809 +soundfile==0.13.1 +astor==0.8.1 +shellingham==1.5.4 +jupyterlab_server==2.27.3 +nvidia-curand-cu12==10.3.7.77 +jupyter-console==6.6.3 +msgpack==1.1.1 +six==1.17.0 +pycparser==2.22 +arrow==1.3.0 +tqdm==4.67.1 +lightning-utilities==0.15.2 +interegular==0.3.3 +rfc3987-syntax==1.1.0 +ipython_pygments_lexers==1.1.1 +datasets==4.0.0 +jupyter_server_terminals==0.5.3 +platformdirs==4.3.8 +pydantic-extra-types==2.10.5 +decorator==5.2.1 +mdurl==0.1.2 +nvidia-cublas-cu12==12.6.4.1 +jsonschema-specifications==2025.4.1 +setuptools==78.1.1 +fastrlock==0.8.3 +argon2-cffi-bindings==25.1.0 +nbconvert==7.16.6 diff --git a/BioReason_new/wandb/run-20250813_153318-ig4rhoqf/files/wandb-metadata.json b/BioReason_new/wandb/run-20250813_153318-ig4rhoqf/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..c54de0858064de2c6cad01df599fe8010e5a004c --- /dev/null +++ b/BioReason_new/wandb/run-20250813_153318-ig4rhoqf/files/wandb-metadata.json @@ -0,0 +1,115 @@ +{ + "os": "Linux-5.10.134-008.18.kangaroo.al8.x86_64-x86_64-with-glibc2.35", + "python": "CPython 3.11.0", + "startedAt": "2025-08-13T07:33:18.645783Z", + "args": [ + "--text_model_name", + "/oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged", + "--protein_model_name", + "/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m", + "--qformer_model_name", + "/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft", + "--num_query_tokens", + "8", + "--train_dataset", + "/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl", + "--valid_dataset", + "/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl", + "--output_dir", + "./contrastive_outputs", + "--num_epochs", + "10", + "--batch_size", + "8", + "--learning_rate", + "1e-4", + "--temperature", + "0.07", + "--freeze_protein_model", + "--freeze_text_model", + "--enable_ptm", + "--max_length_protein", + "1024", + "--max_length_text", + "512", + "--num_workers", + "8", + "--eval_dataset", + "--use_wandb", + "--wandb_project", + "protein-llm-contrastive", + "--logging_steps", + "100", + "--eval_steps", + "500", + "--save_steps", + "1000" + ], + "program": "/nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py", + "codePath": "wangyujia/BioReason_new/train_contrastive.py", + "codePathLocal": "train_contrastive.py", + "git": { + "remote": "https://github.com/PorUna-byte/PAR.git", + "commit": "b8caf406aa1699c788f0ca6e44a1769452c317db" + }, + "root": "/nas/shared/kilab/wangyujia/BioReason_new", + "host": "dsw-265304-585cc9d768-ckfd7", + "executable": "/root/miniconda3/envs/bioreason/bin/python", + "cpu_count": 64, + "cpu_count_logical": 64, + "gpu": "NVIDIA A800-SXM4-80GB", + "gpu_count": 8, + "disk": { + "/": { + "total": "1623302262784", + "used": "1015304192" + } + }, + "memory": { + "total": "549755813888" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-62fcc532-880f-7324-eb13-3b739a38c29b" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-af1197aa-3e7d-a53e-185d-04b1d64164da" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-3e6e551f-2d7d-a27b-654a-da56933d2283" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-a947b64a-c545-ee5e-30e6-b4b0e2dd8fdc" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-ebd04f3f-c437-1b8b-26f2-4b1ebfee93c4" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-53994891-72a6-25be-a4c8-e32c695155b0" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-a927d00a-15c5-31f5-9ae1-9d2e773821a1" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "architecture": "Ampere", + "uuid": "GPU-93d555c4-29a0-cd78-abac-d98cd160d4e1" + } + ], + "cudaVersion": "12.1", + "writerId": "6fva4lshb0nkqoae8k41gurhtr9m5gpr" +} \ No newline at end of file diff --git a/BioReason_new/wandb/run-20250813_153318-ig4rhoqf/files/wandb-summary.json b/BioReason_new/wandb/run-20250813_153318-ig4rhoqf/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..1af1ec3940714bd273a10f70a8eb86a7ec99ff3f --- /dev/null +++ b/BioReason_new/wandb/run-20250813_153318-ig4rhoqf/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":15},"_runtime":15} \ No newline at end of file diff --git a/BioReason_new/wandb/run-20250813_153318-ig4rhoqf/logs/debug-internal.log b/BioReason_new/wandb/run-20250813_153318-ig4rhoqf/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..aa370bae59eeddb0ea72d65419d4db4231e504d8 --- /dev/null +++ b/BioReason_new/wandb/run-20250813_153318-ig4rhoqf/logs/debug-internal.log @@ -0,0 +1,28 @@ +{"time":"2025-08-13T15:33:18.937087932+08:00","level":"INFO","msg":"stream: starting","core version":"0.21.1"} +{"time":"2025-08-13T15:33:22.125246419+08:00","level":"INFO","msg":"stream: created new stream","id":"ig4rhoqf"} +{"time":"2025-08-13T15:33:22.126020019+08:00","level":"INFO","msg":"stream: started","id":"ig4rhoqf"} +{"time":"2025-08-13T15:33:22.12605541+08:00","level":"INFO","msg":"writer: started","stream_id":"ig4rhoqf"} +{"time":"2025-08-13T15:33:22.126066944+08:00","level":"INFO","msg":"handler: started","stream_id":"ig4rhoqf"} +{"time":"2025-08-13T15:33:22.126093203+08:00","level":"INFO","msg":"sender: started","stream_id":"ig4rhoqf"} +{"time":"2025-08-13T15:33:29.266636932+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ig4rhoqf/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T073326Z&X-Goog-Expires=86399&X-Goog-Signature=1c9fbc6cbd51ae71160b439aa8a692f95ee16291fd45e2c33912f90a5ae3a56a09f33e745a416b04da501c1cf40827caa3399f6cde0752ce3f571c12353aa477d27f258d11b74762a029b2fada9797f27a85c7fe03e3bb1ec12fb98789a6b389cf000b2742ca658544ea41f22c9631272e67f2ea275c4a5b7b9c516fed6ed22ccce6b26b684c341e337963a684be9b248bd64b665fb73437d28ce7365767fe6721770c9f8f75e150767d9e4154a983b7f6aafcc31bbc050a26df2e9a3b99e065c111d7a92525568b04cf8d8a7490e6b8f77aed12c26aa378d0708385b0f40b22af3988b5c6cfb25918b1c9471be072bdfdc7a986ca0774f732d27361d23f3448&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:52222->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ig4rhoqf/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T073326Z&X-Goog-Expires=86399&X-Goog-Signature=1c9fbc6cbd51ae71160b439aa8a692f95ee16291fd45e2c33912f90a5ae3a56a09f33e745a416b04da501c1cf40827caa3399f6cde0752ce3f571c12353aa477d27f258d11b74762a029b2fada9797f27a85c7fe03e3bb1ec12fb98789a6b389cf000b2742ca658544ea41f22c9631272e67f2ea275c4a5b7b9c516fed6ed22ccce6b26b684c341e337963a684be9b248bd64b665fb73437d28ce7365767fe6721770c9f8f75e150767d9e4154a983b7f6aafcc31bbc050a26df2e9a3b99e065c111d7a92525568b04cf8d8a7490e6b8f77aed12c26aa378d0708385b0f40b22af3988b5c6cfb25918b1c9471be072bdfdc7a986ca0774f732d27361d23f3448&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:33:31.205628552+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ig4rhoqf/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T073326Z&X-Goog-Expires=86399&X-Goog-Signature=27041959870a9e2ba8f974ca28bb5723ad374fc1917b982522ea3f555344b25fed8f73d5b443663eb1d7d10499d748efa90c8f6d91802daabf9e8c31258a7371e93aca1694d3e8bcb35a72d7c9ca243dc164f5dfae6ec3009c25cc78ea8cf37629f017b7538998f44f7c65ccfb675e343601421475a4490c754c5ee0370d3d5dfa928faddfe9a90621302ce69efd3d26c51f49c23f92148b018281ccd02f22c42e73e318594ea9c2ff9b25ad13163b60c37f1ededb4dc3a50712bcabdffed71883e2e5b5f04c40f13612d9ff1b6762b9f79d6e19873e959fbf9495e4827e901e8dbd25d7b6f291841915bc428212ad0a142cc3a9b7d04dbba9b49873573481fa&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:57666->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ig4rhoqf/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T073326Z&X-Goog-Expires=86399&X-Goog-Signature=27041959870a9e2ba8f974ca28bb5723ad374fc1917b982522ea3f555344b25fed8f73d5b443663eb1d7d10499d748efa90c8f6d91802daabf9e8c31258a7371e93aca1694d3e8bcb35a72d7c9ca243dc164f5dfae6ec3009c25cc78ea8cf37629f017b7538998f44f7c65ccfb675e343601421475a4490c754c5ee0370d3d5dfa928faddfe9a90621302ce69efd3d26c51f49c23f92148b018281ccd02f22c42e73e318594ea9c2ff9b25ad13163b60c37f1ededb4dc3a50712bcabdffed71883e2e5b5f04c40f13612d9ff1b6762b9f79d6e19873e959fbf9495e4827e901e8dbd25d7b6f291841915bc428212ad0a142cc3a9b7d04dbba9b49873573481fa&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:33:31.959506763+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ig4rhoqf/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T073326Z&X-Goog-Expires=86399&X-Goog-Signature=1c9fbc6cbd51ae71160b439aa8a692f95ee16291fd45e2c33912f90a5ae3a56a09f33e745a416b04da501c1cf40827caa3399f6cde0752ce3f571c12353aa477d27f258d11b74762a029b2fada9797f27a85c7fe03e3bb1ec12fb98789a6b389cf000b2742ca658544ea41f22c9631272e67f2ea275c4a5b7b9c516fed6ed22ccce6b26b684c341e337963a684be9b248bd64b665fb73437d28ce7365767fe6721770c9f8f75e150767d9e4154a983b7f6aafcc31bbc050a26df2e9a3b99e065c111d7a92525568b04cf8d8a7490e6b8f77aed12c26aa378d0708385b0f40b22af3988b5c6cfb25918b1c9471be072bdfdc7a986ca0774f732d27361d23f3448&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:52232->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ig4rhoqf/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T073326Z&X-Goog-Expires=86399&X-Goog-Signature=1c9fbc6cbd51ae71160b439aa8a692f95ee16291fd45e2c33912f90a5ae3a56a09f33e745a416b04da501c1cf40827caa3399f6cde0752ce3f571c12353aa477d27f258d11b74762a029b2fada9797f27a85c7fe03e3bb1ec12fb98789a6b389cf000b2742ca658544ea41f22c9631272e67f2ea275c4a5b7b9c516fed6ed22ccce6b26b684c341e337963a684be9b248bd64b665fb73437d28ce7365767fe6721770c9f8f75e150767d9e4154a983b7f6aafcc31bbc050a26df2e9a3b99e065c111d7a92525568b04cf8d8a7490e6b8f77aed12c26aa378d0708385b0f40b22af3988b5c6cfb25918b1c9471be072bdfdc7a986ca0774f732d27361d23f3448&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:33:33.871387522+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ig4rhoqf/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T073326Z&X-Goog-Expires=86399&X-Goog-Signature=27041959870a9e2ba8f974ca28bb5723ad374fc1917b982522ea3f555344b25fed8f73d5b443663eb1d7d10499d748efa90c8f6d91802daabf9e8c31258a7371e93aca1694d3e8bcb35a72d7c9ca243dc164f5dfae6ec3009c25cc78ea8cf37629f017b7538998f44f7c65ccfb675e343601421475a4490c754c5ee0370d3d5dfa928faddfe9a90621302ce69efd3d26c51f49c23f92148b018281ccd02f22c42e73e318594ea9c2ff9b25ad13163b60c37f1ededb4dc3a50712bcabdffed71883e2e5b5f04c40f13612d9ff1b6762b9f79d6e19873e959fbf9495e4827e901e8dbd25d7b6f291841915bc428212ad0a142cc3a9b7d04dbba9b49873573481fa&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:52238->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ig4rhoqf/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T073326Z&X-Goog-Expires=86399&X-Goog-Signature=27041959870a9e2ba8f974ca28bb5723ad374fc1917b982522ea3f555344b25fed8f73d5b443663eb1d7d10499d748efa90c8f6d91802daabf9e8c31258a7371e93aca1694d3e8bcb35a72d7c9ca243dc164f5dfae6ec3009c25cc78ea8cf37629f017b7538998f44f7c65ccfb675e343601421475a4490c754c5ee0370d3d5dfa928faddfe9a90621302ce69efd3d26c51f49c23f92148b018281ccd02f22c42e73e318594ea9c2ff9b25ad13163b60c37f1ededb4dc3a50712bcabdffed71883e2e5b5f04c40f13612d9ff1b6762b9f79d6e19873e959fbf9495e4827e901e8dbd25d7b6f291841915bc428212ad0a142cc3a9b7d04dbba9b49873573481fa&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:33:36.261477525+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ig4rhoqf/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T073326Z&X-Goog-Expires=86399&X-Goog-Signature=1c9fbc6cbd51ae71160b439aa8a692f95ee16291fd45e2c33912f90a5ae3a56a09f33e745a416b04da501c1cf40827caa3399f6cde0752ce3f571c12353aa477d27f258d11b74762a029b2fada9797f27a85c7fe03e3bb1ec12fb98789a6b389cf000b2742ca658544ea41f22c9631272e67f2ea275c4a5b7b9c516fed6ed22ccce6b26b684c341e337963a684be9b248bd64b665fb73437d28ce7365767fe6721770c9f8f75e150767d9e4154a983b7f6aafcc31bbc050a26df2e9a3b99e065c111d7a92525568b04cf8d8a7490e6b8f77aed12c26aa378d0708385b0f40b22af3988b5c6cfb25918b1c9471be072bdfdc7a986ca0774f732d27361d23f3448&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:57678->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ig4rhoqf/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T073326Z&X-Goog-Expires=86399&X-Goog-Signature=1c9fbc6cbd51ae71160b439aa8a692f95ee16291fd45e2c33912f90a5ae3a56a09f33e745a416b04da501c1cf40827caa3399f6cde0752ce3f571c12353aa477d27f258d11b74762a029b2fada9797f27a85c7fe03e3bb1ec12fb98789a6b389cf000b2742ca658544ea41f22c9631272e67f2ea275c4a5b7b9c516fed6ed22ccce6b26b684c341e337963a684be9b248bd64b665fb73437d28ce7365767fe6721770c9f8f75e150767d9e4154a983b7f6aafcc31bbc050a26df2e9a3b99e065c111d7a92525568b04cf8d8a7490e6b8f77aed12c26aa378d0708385b0f40b22af3988b5c6cfb25918b1c9471be072bdfdc7a986ca0774f732d27361d23f3448&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:33:41.039143093+08:00","level":"INFO","msg":"stream: closing","id":"ig4rhoqf"} +{"time":"2025-08-13T15:38:00.686011134+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ig4rhoqf/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T073326Z&X-Goog-Expires=86399&X-Goog-Signature=27041959870a9e2ba8f974ca28bb5723ad374fc1917b982522ea3f555344b25fed8f73d5b443663eb1d7d10499d748efa90c8f6d91802daabf9e8c31258a7371e93aca1694d3e8bcb35a72d7c9ca243dc164f5dfae6ec3009c25cc78ea8cf37629f017b7538998f44f7c65ccfb675e343601421475a4490c754c5ee0370d3d5dfa928faddfe9a90621302ce69efd3d26c51f49c23f92148b018281ccd02f22c42e73e318594ea9c2ff9b25ad13163b60c37f1ededb4dc3a50712bcabdffed71883e2e5b5f04c40f13612d9ff1b6762b9f79d6e19873e959fbf9495e4827e901e8dbd25d7b6f291841915bc428212ad0a142cc3a9b7d04dbba9b49873573481fa&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:44768->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ig4rhoqf/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T073326Z&X-Goog-Expires=86399&X-Goog-Signature=27041959870a9e2ba8f974ca28bb5723ad374fc1917b982522ea3f555344b25fed8f73d5b443663eb1d7d10499d748efa90c8f6d91802daabf9e8c31258a7371e93aca1694d3e8bcb35a72d7c9ca243dc164f5dfae6ec3009c25cc78ea8cf37629f017b7538998f44f7c65ccfb675e343601421475a4490c754c5ee0370d3d5dfa928faddfe9a90621302ce69efd3d26c51f49c23f92148b018281ccd02f22c42e73e318594ea9c2ff9b25ad13163b60c37f1ededb4dc3a50712bcabdffed71883e2e5b5f04c40f13612d9ff1b6762b9f79d6e19873e959fbf9495e4827e901e8dbd25d7b6f291841915bc428212ad0a142cc3a9b7d04dbba9b49873573481fa&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:38:04.732136338+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ig4rhoqf/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T073342Z&X-Goog-Expires=86399&X-Goog-Signature=1d3cd94b4887f4c0adaa644487d39281fc878818fa20fa0c39b307b4d29a0774e1077acff64b5047465d575c9417f96645240e67e6dab6d1cc25f617e622bcf0f8715629cc8509684074bcd0688360af452c0f89dc80a4e9025500a6e0d16485ec7617abfd0ff10c8ae62cf6012c0f43aad8d30bb7c9de79f87c56f73ccabc572e83285debd09c51e5fcc95e1a7fe1cc011606044e056dc5f2d6756e365041194a3e52cc500d0e71499406481acf9cc760c1da018d07a35e07830dd04208781140f1fc38c86f5aa24bf65e1a58d35406ae84e7026fe6cf59353cf16ca53b51a813beb22417486e4fabc77de1aaef6ee20b755ce21b765571f6bfbf8473942f9d&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:44778->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ig4rhoqf/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T073342Z&X-Goog-Expires=86399&X-Goog-Signature=1d3cd94b4887f4c0adaa644487d39281fc878818fa20fa0c39b307b4d29a0774e1077acff64b5047465d575c9417f96645240e67e6dab6d1cc25f617e622bcf0f8715629cc8509684074bcd0688360af452c0f89dc80a4e9025500a6e0d16485ec7617abfd0ff10c8ae62cf6012c0f43aad8d30bb7c9de79f87c56f73ccabc572e83285debd09c51e5fcc95e1a7fe1cc011606044e056dc5f2d6756e365041194a3e52cc500d0e71499406481acf9cc760c1da018d07a35e07830dd04208781140f1fc38c86f5aa24bf65e1a58d35406ae84e7026fe6cf59353cf16ca53b51a813beb22417486e4fabc77de1aaef6ee20b755ce21b765571f6bfbf8473942f9d&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:38:04.732752873+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ig4rhoqf/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T073342Z&X-Goog-Expires=86399&X-Goog-Signature=43378ba7835541b25e7884ddd1295c99cdb59af218b575e11ebf49219cd7b70414883b2eaabf6ce5bf29b6dcf3aa4854cc71ac3f00aceb098b2bd9f008cbf3c3e776aeb0eec09eafbd1633dbdf5cd30ca267e7fc78a69846132727fb83834b13e3aaf2e17cf6af4974dc4f96f982c762fbd056b369608aece84f3c97eed6c02b7648d28a1ab3f3566026c830a6eb4d57a55676a251768961021c9050a7b6974e74c7efd13ca9324ac1a6768bca2daa0daf12b3c8ebc3ad02e5d5c935024048da77f0b6047842ebceaf42ec881dd23660f5f4caad61c04ea8736f4b8abd5c81197ebd10853a45e4944b85d318500c385596344922b43e114698bc05474aad582c&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:43548->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ig4rhoqf/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T073342Z&X-Goog-Expires=86399&X-Goog-Signature=43378ba7835541b25e7884ddd1295c99cdb59af218b575e11ebf49219cd7b70414883b2eaabf6ce5bf29b6dcf3aa4854cc71ac3f00aceb098b2bd9f008cbf3c3e776aeb0eec09eafbd1633dbdf5cd30ca267e7fc78a69846132727fb83834b13e3aaf2e17cf6af4974dc4f96f982c762fbd056b369608aece84f3c97eed6c02b7648d28a1ab3f3566026c830a6eb4d57a55676a251768961021c9050a7b6974e74c7efd13ca9324ac1a6768bca2daa0daf12b3c8ebc3ad02e5d5c935024048da77f0b6047842ebceaf42ec881dd23660f5f4caad61c04ea8736f4b8abd5c81197ebd10853a45e4944b85d318500c385596344922b43e114698bc05474aad582c&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:38:04.830072606+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ig4rhoqf/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T073342Z&X-Goog-Expires=86399&X-Goog-Signature=c0195bb166fc1589772f7ac8000fd444b8e06323ba6372e465965830037d0e3abe33eb458ac66e538461c44cd0aad54fae2cf3ed0f374b29b4cc12eefa1bd24dbcc6dd7346e5d2fe8238aca5f9f454f181d5839a406a34dc8e519e37ef8c4fc8de26b075401ceec8431d3f6c56345d424b5cace6503127369284bda44efb6fe17a73b78fc162b5452942692cd51024f02807e743229ac8d7ddee751592f125d012c3e4de0790abc7b061c8d8d6916a275902955479dcddb6342607f857379f4552e01426a19bf54b2231de5fb22f340330a26d23e6cae7d90831b2172a8cfdd004a5de1aca9d47db2eded16ebcbeb22d617adfb1a0324e51847993c2374681dd&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:43546->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ig4rhoqf/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T073342Z&X-Goog-Expires=86399&X-Goog-Signature=c0195bb166fc1589772f7ac8000fd444b8e06323ba6372e465965830037d0e3abe33eb458ac66e538461c44cd0aad54fae2cf3ed0f374b29b4cc12eefa1bd24dbcc6dd7346e5d2fe8238aca5f9f454f181d5839a406a34dc8e519e37ef8c4fc8de26b075401ceec8431d3f6c56345d424b5cace6503127369284bda44efb6fe17a73b78fc162b5452942692cd51024f02807e743229ac8d7ddee751592f125d012c3e4de0790abc7b061c8d8d6916a275902955479dcddb6342607f857379f4552e01426a19bf54b2231de5fb22f340330a26d23e6cae7d90831b2172a8cfdd004a5de1aca9d47db2eded16ebcbeb22d617adfb1a0324e51847993c2374681dd&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:38:08.914871723+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ig4rhoqf/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T073326Z&X-Goog-Expires=86399&X-Goog-Signature=1c9fbc6cbd51ae71160b439aa8a692f95ee16291fd45e2c33912f90a5ae3a56a09f33e745a416b04da501c1cf40827caa3399f6cde0752ce3f571c12353aa477d27f258d11b74762a029b2fada9797f27a85c7fe03e3bb1ec12fb98789a6b389cf000b2742ca658544ea41f22c9631272e67f2ea275c4a5b7b9c516fed6ed22ccce6b26b684c341e337963a684be9b248bd64b665fb73437d28ce7365767fe6721770c9f8f75e150767d9e4154a983b7f6aafcc31bbc050a26df2e9a3b99e065c111d7a92525568b04cf8d8a7490e6b8f77aed12c26aa378d0708385b0f40b22af3988b5c6cfb25918b1c9471be072bdfdc7a986ca0774f732d27361d23f3448&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:40682->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ig4rhoqf/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T073326Z&X-Goog-Expires=86399&X-Goog-Signature=1c9fbc6cbd51ae71160b439aa8a692f95ee16291fd45e2c33912f90a5ae3a56a09f33e745a416b04da501c1cf40827caa3399f6cde0752ce3f571c12353aa477d27f258d11b74762a029b2fada9797f27a85c7fe03e3bb1ec12fb98789a6b389cf000b2742ca658544ea41f22c9631272e67f2ea275c4a5b7b9c516fed6ed22ccce6b26b684c341e337963a684be9b248bd64b665fb73437d28ce7365767fe6721770c9f8f75e150767d9e4154a983b7f6aafcc31bbc050a26df2e9a3b99e065c111d7a92525568b04cf8d8a7490e6b8f77aed12c26aa378d0708385b0f40b22af3988b5c6cfb25918b1c9471be072bdfdc7a986ca0774f732d27361d23f3448&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:38:09.512888171+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ig4rhoqf/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T073326Z&X-Goog-Expires=86399&X-Goog-Signature=27041959870a9e2ba8f974ca28bb5723ad374fc1917b982522ea3f555344b25fed8f73d5b443663eb1d7d10499d748efa90c8f6d91802daabf9e8c31258a7371e93aca1694d3e8bcb35a72d7c9ca243dc164f5dfae6ec3009c25cc78ea8cf37629f017b7538998f44f7c65ccfb675e343601421475a4490c754c5ee0370d3d5dfa928faddfe9a90621302ce69efd3d26c51f49c23f92148b018281ccd02f22c42e73e318594ea9c2ff9b25ad13163b60c37f1ededb4dc3a50712bcabdffed71883e2e5b5f04c40f13612d9ff1b6762b9f79d6e19873e959fbf9495e4827e901e8dbd25d7b6f291841915bc428212ad0a142cc3a9b7d04dbba9b49873573481fa&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:53738->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ig4rhoqf/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T073326Z&X-Goog-Expires=86399&X-Goog-Signature=27041959870a9e2ba8f974ca28bb5723ad374fc1917b982522ea3f555344b25fed8f73d5b443663eb1d7d10499d748efa90c8f6d91802daabf9e8c31258a7371e93aca1694d3e8bcb35a72d7c9ca243dc164f5dfae6ec3009c25cc78ea8cf37629f017b7538998f44f7c65ccfb675e343601421475a4490c754c5ee0370d3d5dfa928faddfe9a90621302ce69efd3d26c51f49c23f92148b018281ccd02f22c42e73e318594ea9c2ff9b25ad13163b60c37f1ededb4dc3a50712bcabdffed71883e2e5b5f04c40f13612d9ff1b6762b9f79d6e19873e959fbf9495e4827e901e8dbd25d7b6f291841915bc428212ad0a142cc3a9b7d04dbba9b49873573481fa&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:38:25.342508911+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ig4rhoqf/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T073326Z&X-Goog-Expires=86399&X-Goog-Signature=1c9fbc6cbd51ae71160b439aa8a692f95ee16291fd45e2c33912f90a5ae3a56a09f33e745a416b04da501c1cf40827caa3399f6cde0752ce3f571c12353aa477d27f258d11b74762a029b2fada9797f27a85c7fe03e3bb1ec12fb98789a6b389cf000b2742ca658544ea41f22c9631272e67f2ea275c4a5b7b9c516fed6ed22ccce6b26b684c341e337963a684be9b248bd64b665fb73437d28ce7365767fe6721770c9f8f75e150767d9e4154a983b7f6aafcc31bbc050a26df2e9a3b99e065c111d7a92525568b04cf8d8a7490e6b8f77aed12c26aa378d0708385b0f40b22af3988b5c6cfb25918b1c9471be072bdfdc7a986ca0774f732d27361d23f3448&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:49916->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ig4rhoqf/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T073326Z&X-Goog-Expires=86399&X-Goog-Signature=1c9fbc6cbd51ae71160b439aa8a692f95ee16291fd45e2c33912f90a5ae3a56a09f33e745a416b04da501c1cf40827caa3399f6cde0752ce3f571c12353aa477d27f258d11b74762a029b2fada9797f27a85c7fe03e3bb1ec12fb98789a6b389cf000b2742ca658544ea41f22c9631272e67f2ea275c4a5b7b9c516fed6ed22ccce6b26b684c341e337963a684be9b248bd64b665fb73437d28ce7365767fe6721770c9f8f75e150767d9e4154a983b7f6aafcc31bbc050a26df2e9a3b99e065c111d7a92525568b04cf8d8a7490e6b8f77aed12c26aa378d0708385b0f40b22af3988b5c6cfb25918b1c9471be072bdfdc7a986ca0774f732d27361d23f3448&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:38:59.014612034+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ig4rhoqf/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T073326Z&X-Goog-Expires=86399&X-Goog-Signature=1c9fbc6cbd51ae71160b439aa8a692f95ee16291fd45e2c33912f90a5ae3a56a09f33e745a416b04da501c1cf40827caa3399f6cde0752ce3f571c12353aa477d27f258d11b74762a029b2fada9797f27a85c7fe03e3bb1ec12fb98789a6b389cf000b2742ca658544ea41f22c9631272e67f2ea275c4a5b7b9c516fed6ed22ccce6b26b684c341e337963a684be9b248bd64b665fb73437d28ce7365767fe6721770c9f8f75e150767d9e4154a983b7f6aafcc31bbc050a26df2e9a3b99e065c111d7a92525568b04cf8d8a7490e6b8f77aed12c26aa378d0708385b0f40b22af3988b5c6cfb25918b1c9471be072bdfdc7a986ca0774f732d27361d23f3448&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:44582->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ig4rhoqf/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T073326Z&X-Goog-Expires=86399&X-Goog-Signature=1c9fbc6cbd51ae71160b439aa8a692f95ee16291fd45e2c33912f90a5ae3a56a09f33e745a416b04da501c1cf40827caa3399f6cde0752ce3f571c12353aa477d27f258d11b74762a029b2fada9797f27a85c7fe03e3bb1ec12fb98789a6b389cf000b2742ca658544ea41f22c9631272e67f2ea275c4a5b7b9c516fed6ed22ccce6b26b684c341e337963a684be9b248bd64b665fb73437d28ce7365767fe6721770c9f8f75e150767d9e4154a983b7f6aafcc31bbc050a26df2e9a3b99e065c111d7a92525568b04cf8d8a7490e6b8f77aed12c26aa378d0708385b0f40b22af3988b5c6cfb25918b1c9471be072bdfdc7a986ca0774f732d27361d23f3448&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:40:15.762761262+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ig4rhoqf/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T073342Z&X-Goog-Expires=86399&X-Goog-Signature=1d3cd94b4887f4c0adaa644487d39281fc878818fa20fa0c39b307b4d29a0774e1077acff64b5047465d575c9417f96645240e67e6dab6d1cc25f617e622bcf0f8715629cc8509684074bcd0688360af452c0f89dc80a4e9025500a6e0d16485ec7617abfd0ff10c8ae62cf6012c0f43aad8d30bb7c9de79f87c56f73ccabc572e83285debd09c51e5fcc95e1a7fe1cc011606044e056dc5f2d6756e365041194a3e52cc500d0e71499406481acf9cc760c1da018d07a35e07830dd04208781140f1fc38c86f5aa24bf65e1a58d35406ae84e7026fe6cf59353cf16ca53b51a813beb22417486e4fabc77de1aaef6ee20b755ce21b765571f6bfbf8473942f9d&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:46682->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ig4rhoqf/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T073342Z&X-Goog-Expires=86399&X-Goog-Signature=1d3cd94b4887f4c0adaa644487d39281fc878818fa20fa0c39b307b4d29a0774e1077acff64b5047465d575c9417f96645240e67e6dab6d1cc25f617e622bcf0f8715629cc8509684074bcd0688360af452c0f89dc80a4e9025500a6e0d16485ec7617abfd0ff10c8ae62cf6012c0f43aad8d30bb7c9de79f87c56f73ccabc572e83285debd09c51e5fcc95e1a7fe1cc011606044e056dc5f2d6756e365041194a3e52cc500d0e71499406481acf9cc760c1da018d07a35e07830dd04208781140f1fc38c86f5aa24bf65e1a58d35406ae84e7026fe6cf59353cf16ca53b51a813beb22417486e4fabc77de1aaef6ee20b755ce21b765571f6bfbf8473942f9d&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:40:15.7960472+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ig4rhoqf/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T073342Z&X-Goog-Expires=86399&X-Goog-Signature=43378ba7835541b25e7884ddd1295c99cdb59af218b575e11ebf49219cd7b70414883b2eaabf6ce5bf29b6dcf3aa4854cc71ac3f00aceb098b2bd9f008cbf3c3e776aeb0eec09eafbd1633dbdf5cd30ca267e7fc78a69846132727fb83834b13e3aaf2e17cf6af4974dc4f96f982c762fbd056b369608aece84f3c97eed6c02b7648d28a1ab3f3566026c830a6eb4d57a55676a251768961021c9050a7b6974e74c7efd13ca9324ac1a6768bca2daa0daf12b3c8ebc3ad02e5d5c935024048da77f0b6047842ebceaf42ec881dd23660f5f4caad61c04ea8736f4b8abd5c81197ebd10853a45e4944b85d318500c385596344922b43e114698bc05474aad582c&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:46696->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ig4rhoqf/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T073342Z&X-Goog-Expires=86399&X-Goog-Signature=43378ba7835541b25e7884ddd1295c99cdb59af218b575e11ebf49219cd7b70414883b2eaabf6ce5bf29b6dcf3aa4854cc71ac3f00aceb098b2bd9f008cbf3c3e776aeb0eec09eafbd1633dbdf5cd30ca267e7fc78a69846132727fb83834b13e3aaf2e17cf6af4974dc4f96f982c762fbd056b369608aece84f3c97eed6c02b7648d28a1ab3f3566026c830a6eb4d57a55676a251768961021c9050a7b6974e74c7efd13ca9324ac1a6768bca2daa0daf12b3c8ebc3ad02e5d5c935024048da77f0b6047842ebceaf42ec881dd23660f5f4caad61c04ea8736f4b8abd5c81197ebd10853a45e4944b85d318500c385596344922b43e114698bc05474aad582c&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:40:15.806561829+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ig4rhoqf/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T073342Z&X-Goog-Expires=86399&X-Goog-Signature=c0195bb166fc1589772f7ac8000fd444b8e06323ba6372e465965830037d0e3abe33eb458ac66e538461c44cd0aad54fae2cf3ed0f374b29b4cc12eefa1bd24dbcc6dd7346e5d2fe8238aca5f9f454f181d5839a406a34dc8e519e37ef8c4fc8de26b075401ceec8431d3f6c56345d424b5cace6503127369284bda44efb6fe17a73b78fc162b5452942692cd51024f02807e743229ac8d7ddee751592f125d012c3e4de0790abc7b061c8d8d6916a275902955479dcddb6342607f857379f4552e01426a19bf54b2231de5fb22f340330a26d23e6cae7d90831b2172a8cfdd004a5de1aca9d47db2eded16ebcbeb22d617adfb1a0324e51847993c2374681dd&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:46686->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ig4rhoqf/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T073342Z&X-Goog-Expires=86399&X-Goog-Signature=c0195bb166fc1589772f7ac8000fd444b8e06323ba6372e465965830037d0e3abe33eb458ac66e538461c44cd0aad54fae2cf3ed0f374b29b4cc12eefa1bd24dbcc6dd7346e5d2fe8238aca5f9f454f181d5839a406a34dc8e519e37ef8c4fc8de26b075401ceec8431d3f6c56345d424b5cace6503127369284bda44efb6fe17a73b78fc162b5452942692cd51024f02807e743229ac8d7ddee751592f125d012c3e4de0790abc7b061c8d8d6916a275902955479dcddb6342607f857379f4552e01426a19bf54b2231de5fb22f340330a26d23e6cae7d90831b2172a8cfdd004a5de1aca9d47db2eded16ebcbeb22d617adfb1a0324e51847993c2374681dd&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:40:20.882189315+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ig4rhoqf/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T073342Z&X-Goog-Expires=86399&X-Goog-Signature=c0195bb166fc1589772f7ac8000fd444b8e06323ba6372e465965830037d0e3abe33eb458ac66e538461c44cd0aad54fae2cf3ed0f374b29b4cc12eefa1bd24dbcc6dd7346e5d2fe8238aca5f9f454f181d5839a406a34dc8e519e37ef8c4fc8de26b075401ceec8431d3f6c56345d424b5cace6503127369284bda44efb6fe17a73b78fc162b5452942692cd51024f02807e743229ac8d7ddee751592f125d012c3e4de0790abc7b061c8d8d6916a275902955479dcddb6342607f857379f4552e01426a19bf54b2231de5fb22f340330a26d23e6cae7d90831b2172a8cfdd004a5de1aca9d47db2eded16ebcbeb22d617adfb1a0324e51847993c2374681dd&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.1.90:55506->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/ig4rhoqf/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250813%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250813T073342Z&X-Goog-Expires=86399&X-Goog-Signature=c0195bb166fc1589772f7ac8000fd444b8e06323ba6372e465965830037d0e3abe33eb458ac66e538461c44cd0aad54fae2cf3ed0f374b29b4cc12eefa1bd24dbcc6dd7346e5d2fe8238aca5f9f454f181d5839a406a34dc8e519e37ef8c4fc8de26b075401ceec8431d3f6c56345d424b5cace6503127369284bda44efb6fe17a73b78fc162b5452942692cd51024f02807e743229ac8d7ddee751592f125d012c3e4de0790abc7b061c8d8d6916a275902955479dcddb6342607f857379f4552e01426a19bf54b2231de5fb22f340330a26d23e6cae7d90831b2172a8cfdd004a5de1aca9d47db2eded16ebcbeb22d617adfb1a0324e51847993c2374681dd&X-Goog-SignedHeaders=host&X-User=gia0603yucca"} +{"time":"2025-08-13T15:42:32.350288869+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-08-13T15:42:37.496789088+08:00","level":"INFO","msg":"handler: closed","stream_id":"ig4rhoqf"} +{"time":"2025-08-13T15:42:37.502696559+08:00","level":"INFO","msg":"sender: closed","stream_id":"ig4rhoqf"} +{"time":"2025-08-13T15:42:37.502726375+08:00","level":"INFO","msg":"stream: closed","id":"ig4rhoqf"} diff --git a/BioReason_new/wandb/run-20250813_153318-ig4rhoqf/logs/debug.log b/BioReason_new/wandb/run-20250813_153318-ig4rhoqf/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..341bf808260a271c35ea067134d8f263ef889f33 --- /dev/null +++ b/BioReason_new/wandb/run-20250813_153318-ig4rhoqf/logs/debug.log @@ -0,0 +1,23 @@ +2025-08-13 15:33:18,716 INFO MainThread:13510 [wandb_setup.py:_flush():80] Current SDK version is 0.21.1 +2025-08-13 15:33:18,716 INFO MainThread:13510 [wandb_setup.py:_flush():80] Configure stats pid to 13510 +2025-08-13 15:33:18,716 INFO MainThread:13510 [wandb_setup.py:_flush():80] Loading settings from /root/.config/wandb/settings +2025-08-13 15:33:18,716 INFO MainThread:13510 [wandb_setup.py:_flush():80] Loading settings from /nas/shared/kilab/wangyujia/BioReason_new/wandb/settings +2025-08-13 15:33:18,716 INFO MainThread:13510 [wandb_setup.py:_flush():80] Loading settings from environment variables +2025-08-13 15:33:18,716 INFO MainThread:13510 [wandb_init.py:setup_run_log_directory():703] Logging user logs to /nas/shared/kilab/wangyujia/BioReason_new/wandb/run-20250813_153318-ig4rhoqf/logs/debug.log +2025-08-13 15:33:18,716 INFO MainThread:13510 [wandb_init.py:setup_run_log_directory():704] Logging internal logs to /nas/shared/kilab/wangyujia/BioReason_new/wandb/run-20250813_153318-ig4rhoqf/logs/debug-internal.log +2025-08-13 15:33:18,716 INFO MainThread:13510 [wandb_init.py:init():830] calling init triggers +2025-08-13 15:33:18,716 INFO MainThread:13510 [wandb_init.py:init():835] wandb.init called with sweep_config: {} +config: {'text_model_name': '/oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged', 'protein_model_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m', 'qformer_model_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft', 'cache_dir': '/model-weights', 'num_query_tokens': 8, 'train_dataset': '/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl', 'valid_dataset': '/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl', 'eval_dataset': True, 'output_dir': './contrastive_outputs', 'num_epochs': 10, 'batch_size': 8, 'learning_rate': 0.0001, 'weight_decay': 0.01, 'warmup_steps': 1000, 'gradient_accumulation_steps': 1, 'temperature': 0.07, 'freeze_protein_model': True, 'freeze_text_model': True, 'protein_weight': 1.0, 'text_weight': 1.0, 'enable_ptm': True, 'ptm_weight': 1.0, 'max_length_protein': 1024, 'max_length_text': 512, 'num_workers': 8, 'logging_steps': 100, 'eval_steps': 500, 'save_steps': 1000, 'save_total_limit': 3, 'fp16': False, 'bf16': False, 'seed': 42, 'use_wandb': True, 'wandb_project': 'protein-llm-contrastive', 'wandb_entity': None, '_wandb': {}} +2025-08-13 15:33:18,716 INFO MainThread:13510 [wandb_init.py:init():871] starting backend +2025-08-13 15:33:18,925 INFO MainThread:13510 [wandb_init.py:init():874] sending inform_init request +2025-08-13 15:33:18,931 INFO MainThread:13510 [wandb_init.py:init():882] backend started and connected +2025-08-13 15:33:18,933 INFO MainThread:13510 [wandb_init.py:init():953] updated telemetry +2025-08-13 15:33:18,961 INFO MainThread:13510 [wandb_init.py:init():977] communicating run to backend with 90.0 second timeout +2025-08-13 15:33:25,902 INFO MainThread:13510 [wandb_init.py:init():1029] starting run threads in backend +2025-08-13 15:33:26,011 INFO MainThread:13510 [wandb_run.py:_console_start():2494] atexit reg +2025-08-13 15:33:26,011 INFO MainThread:13510 [wandb_run.py:_redirect():2342] redirect: wrap_raw +2025-08-13 15:33:26,011 INFO MainThread:13510 [wandb_run.py:_redirect():2411] Wrapping output streams. +2025-08-13 15:33:26,011 INFO MainThread:13510 [wandb_run.py:_redirect():2434] Redirects installed. +2025-08-13 15:33:26,014 INFO MainThread:13510 [wandb_init.py:init():1075] run started, returning control to user process +2025-08-13 15:33:36,032 INFO MainThread:13510 [wandb_run.py:_config_callback():1380] config_cb None None {'output_dir': './contrastive_outputs', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': True, 'do_predict': False, 'eval_strategy': 'steps', 'prediction_loss_only': False, 'per_device_train_batch_size': 8, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'eval_delay': 0, 'torch_empty_cache_steps': None, 'learning_rate': 0.0001, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 10, 'max_steps': -1, 'lr_scheduler_type': 'linear', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.0, 'warmup_steps': 1000, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': './contrastive_outputs/runs/Aug13_15-33-30_dsw-265304-585cc9d768-ckfd7', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 100, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 1000, 'save_total_limit': 3, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'restore_callback_states_from_checkpoint': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 42, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': False, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 500, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': None, 'disable_tqdm': False, 'remove_unused_columns': False, 'label_names': None, 'load_best_model_at_end': True, 'metric_for_best_model': 'eval_avg_recall_at_1', 'greater_is_better': True, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'deepspeed': None, 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['wandb'], 'ddp_find_unused_parameters': False, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '', 'hub_private_repo': None, 'hub_always_push': False, 'hub_revision': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'include_for_metrics': [], 'eval_do_concat_batches': True, 'fp16_backend': 'auto', 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None, 'batch_eval_metrics': False, 'eval_on_start': False, 'use_liger_kernel': False, 'liger_kernel_config': None, 'eval_use_gather_object': False, 'average_tokens_across_devices': False, 'temperature': 0.07, 'freeze_protein_model': True, 'freeze_text_model': True, 'protein_weight': 1.0, 'text_weight': 1.0, 'max_length_protein': 1024, 'max_length_text': 512, 'enable_ptm': True, 'ptm_weight': 1.0} +2025-08-13 15:33:41,038 INFO MsgRouterThr:13510 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 2 handles. diff --git a/BioReason_new/wandb/run-20250813_153318-ig4rhoqf/run-ig4rhoqf.wandb b/BioReason_new/wandb/run-20250813_153318-ig4rhoqf/run-ig4rhoqf.wandb new file mode 100644 index 0000000000000000000000000000000000000000..32c17a2c5bc3d191db85bc9f6de8854f62dfa7c0 Binary files /dev/null and b/BioReason_new/wandb/run-20250813_153318-ig4rhoqf/run-ig4rhoqf.wandb differ diff --git a/EasyR1/.github/CODE_OF_CONDUCT.md b/EasyR1/.github/CODE_OF_CONDUCT.md new file mode 100644 index 0000000000000000000000000000000000000000..c2035cea5425b8de8e88a563214d05dfd415352a --- /dev/null +++ b/EasyR1/.github/CODE_OF_CONDUCT.md @@ -0,0 +1,128 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity +and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the + overall community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or + advances of any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email + address, without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +`hoshihiyouga AT gmail DOT com`. +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series +of actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or +permanent ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within +the community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.0, available at +https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. + +Community Impact Guidelines were inspired by [Mozilla's code of conduct +enforcement ladder](https://github.com/mozilla/diversity). + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see the FAQ at +https://www.contributor-covenant.org/faq. Translations are available at +https://www.contributor-covenant.org/translations. diff --git a/EasyR1/.github/CONTRIBUTING.md b/EasyR1/.github/CONTRIBUTING.md new file mode 100644 index 0000000000000000000000000000000000000000..75dd70981d538ab46076e180fee68d4dc0cd0d13 --- /dev/null +++ b/EasyR1/.github/CONTRIBUTING.md @@ -0,0 +1,64 @@ +# Contributing to EasyR1 + +Everyone is welcome to contribute, and we value everybody's contribution. Code contributions are not the only way to help the community. Answering questions, helping others, and improving the documentation are also immensely valuable. + +It also helps us if you spread the word! Reference the library in blog posts about the awesome projects it made possible, shout out on Twitter every time it has helped you, or simply ⭐️ the repository to say thank you. + +However you choose to contribute, please be mindful and respect our [code of conduct](CODE_OF_CONDUCT.md). + +**This guide was heavily inspired by [transformers guide to contributing](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md).** + +## Ways to contribute + +There are several ways you can contribute to EasyR1: + +* Fix outstanding issues with the existing code. +* Submit issues related to bugs or desired new features. +* Contribute to the examples or to the documentation. + +### Style guide + +EasyR1 follows the [Google Python Style Guide](https://google.github.io/styleguide/pyguide.html), check it for details. + +### Create a Pull Request + +1. Fork the [repository](https://github.com/hiyouga/EasyR1) by clicking on the [Fork](https://github.com/hiyouga/EasyR1/fork) button on the repository's page. This creates a copy of the code under your GitHub user account. + +2. Clone your fork to your local disk, and add the base repository as a remote: + +```bash +git clone git@github.com:[username]/EasyR1.git +cd EasyR1 +git remote add upstream https://github.com/hiyouga/EasyR1.git +``` + +3. Create a new branch to hold your development changes: + +```bash +git checkout -b dev_your_branch +``` + +4. Set up a development environment by running the following command in a virtual environment: + +```bash +pip install -e ".[dev]" +``` + +5. Check code before commit: + +```bash +make commit +make style && make quality +``` + +6. Submit changes: + +```bash +git add . +git commit -m "commit message" +git fetch upstream +git rebase upstream/main +git push -u origin dev_your_branch +``` + +7. Create a merge request from your branch `dev_your_branch` at [origin repo](https://github.com/hiyouga/EasyR1). diff --git a/EasyR1/.github/requirements-test.txt b/EasyR1/.github/requirements-test.txt new file mode 100644 index 0000000000000000000000000000000000000000..04dd11228334649cf1b6f8aa1ab8e5dee54965a5 --- /dev/null +++ b/EasyR1/.github/requirements-test.txt @@ -0,0 +1,8 @@ +codetiming +datasets +pillow +pytest +ruff +torch +torchvision +transformers diff --git a/EasyR1/.github/workflows/tests.yml b/EasyR1/.github/workflows/tests.yml new file mode 100644 index 0000000000000000000000000000000000000000..90752f1f5316b1bdb940da257591191ecd1d833a --- /dev/null +++ b/EasyR1/.github/workflows/tests.yml @@ -0,0 +1,64 @@ +name: tests + +on: + workflow_dispatch: + push: + branches: + - "main" + paths: + - "**.py" + - "requirements.txt" + - ".github/workflows/*.yml" + pull_request: + branches: + - "main" + paths: + - "**.py" + - "requirements.txt" + - ".github/workflows/*.yml" + +jobs: + tests: + strategy: + fail-fast: false + matrix: + python-version: + - "3.11" + os: + - "ubuntu-latest" + + runs-on: ${{ matrix.os }} + + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.os }}-${{ matrix.python-version }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: "pip" + cache-dependency-path: "**/requirements*.txt" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu + python -m pip install -r .github/requirements-test.txt + python -m pip install --no-deps . + + - name: Check quality + run: | + make style && make quality + + - name: Check license + run: | + make license + + - name: Test with pytest + run: | + make test diff --git a/EasyR1/.gitignore b/EasyR1/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..8a30dcf8a838ae478aef1cc10d4cd0ca300ef701 --- /dev/null +++ b/EasyR1/.gitignore @@ -0,0 +1,181 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# UV +# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +#uv.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/latest/usage/project/#working-with-version-control +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +# PyPI configuration file +.pypirc + +# outputs +outputs/ +checkpoints/ +wandb/ +tensorboard_log/ + +# data +images/ +images* diff --git a/EasyR1/.pre-commit-config.yaml b/EasyR1/.pre-commit-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d91faaf4d2ba7969d63ff9a40050b6c6fd34c23c --- /dev/null +++ b/EasyR1/.pre-commit-config.yaml @@ -0,0 +1,22 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: check-ast + - id: check-added-large-files + args: ['--maxkb=25000'] + - id: check-merge-conflict + - id: check-yaml + - id: debug-statements + - id: end-of-file-fixer + - id: requirements-txt-fixer + - id: trailing-whitespace + args: [--markdown-linebreak-ext=md] + - id: no-commit-to-branch + args: ['--branch', 'main'] + +- repo: https://github.com/asottile/pyupgrade + rev: v3.17.0 + hooks: + - id: pyupgrade + args: [--py38-plus] diff --git a/EasyR1/Dockerfile b/EasyR1/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..8b22d8465ca6eb5291b20c913e21dde3d42ddbd2 --- /dev/null +++ b/EasyR1/Dockerfile @@ -0,0 +1,76 @@ +# Start from the NVIDIA official image (ubuntu-22.04 + cuda-12.6 + python-3.10) +# https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-24-08.html +FROM nvcr.io/nvidia/pytorch:24.08-py3 + +# Define environments +ENV MAX_JOBS=32 +ENV VLLM_WORKER_MULTIPROC_METHOD=spawn +ENV DEBIAN_FRONTEND=noninteractive +ENV NODE_OPTIONS="" +ENV PIP_ROOT_USER_ACTION=ignore +ENV HF_HUB_ENABLE_HF_TRANSFER="1" + +# Define installation arguments +ARG APT_SOURCE=https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ +ARG PIP_INDEX=https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple + +# Set apt source +RUN cp /etc/apt/sources.list /etc/apt/sources.list.bak && \ + { \ + echo "deb ${APT_SOURCE} jammy main restricted universe multiverse"; \ + echo "deb ${APT_SOURCE} jammy-updates main restricted universe multiverse"; \ + echo "deb ${APT_SOURCE} jammy-backports main restricted universe multiverse"; \ + echo "deb ${APT_SOURCE} jammy-security main restricted universe multiverse"; \ + } > /etc/apt/sources.list + +# Install systemctl +RUN apt-get update && \ + apt-get install -y -o Dpkg::Options::="--force-confdef" systemd && \ + apt-get clean + +# Install tini +RUN apt-get update && \ + apt-get install -y tini && \ + apt-get clean + +# Change pip source +RUN pip config set global.index-url "${PIP_INDEX}" && \ + pip config set global.extra-index-url "${PIP_INDEX}" && \ + python -m pip install --upgrade pip + +# Uninstall nv-pytorch fork +RUN pip uninstall -y torch torchvision torchaudio \ + pytorch-quantization pytorch-triton torch-tensorrt \ + transformer_engine flash_attn apex megatron-core \ + xgboost opencv grpcio + +# Fix cv2 +RUN rm -rf /usr/local/lib/python3.10/dist-packages/cv2 + +# Install torch-2.6.0+cu124 + vllm-0.8.4 +# torch-2.6.0+cu124: cxx11abi=False +# torch-2.6.0+cu126: cxx11abi=True +# see https://github.com/flashinfer-ai/flashinfer/issues/911 +RUN pip install --no-cache-dir "vllm==0.8.4" "torch==2.6.0" "torchvision==0.21.0" "torchaudio==2.6.0" tensordict torchdata \ + "transformers[hf_xet]>=4.51.0" accelerate datasets peft hf-transfer \ + "numpy<2.0.0" "pyarrow>=15.0.0" pandas \ + ray[default] codetiming hydra-core pylatexenc qwen-vl-utils wandb liger-kernel mathruler \ + pytest yapf py-spy pyext pre-commit ruff + +# Install flash-attn-2.7.4.post1 (cxx11abi=False) +RUN wget -nv https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiFALSE-cp310-cp310-linux_x86_64.whl && \ + pip install --no-cache-dir flash_attn-2.7.4.post1+cu12torch2.6cxx11abiFALSE-cp310-cp310-linux_x86_64.whl + +# Install flashinfer-0.2.2.post1+cu124 (cxx11abi=False) +# vllm-0.8.3 does not support flashinfer>=0.2.3 +# see https://github.com/vllm-project/vllm/pull/15777 +RUN wget -nv https://github.com/flashinfer-ai/flashinfer/releases/download/v0.2.2.post1/flashinfer_python-0.2.2.post1+cu124torch2.6-cp38-abi3-linux_x86_64.whl && \ + pip install --no-cache-dir flashinfer_python-0.2.2.post1+cu124torch2.6-cp38-abi3-linux_x86_64.whl + +# Fix packages +RUN pip uninstall -y pynvml nvidia-ml-py && \ + pip install --no-cache-dir --upgrade "nvidia-ml-py>=12.560.30" "fastapi[standard]>=0.115.0" "optree>=0.13.0" "pydantic>=2.9" "grpcio>=1.62.1" + +# Reset pip config +RUN pip config unset global.index-url && \ + pip config unset global.extra-index-url diff --git a/EasyR1/Dockerfile.legacy b/EasyR1/Dockerfile.legacy new file mode 100644 index 0000000000000000000000000000000000000000..549598ef3b63b17d48f9935bc0edee26e26ec605 --- /dev/null +++ b/EasyR1/Dockerfile.legacy @@ -0,0 +1,72 @@ +# Start from the NVIDIA official image (ubuntu-22.04 + cuda-12.6 + python-3.10) +# https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-24-08.html +FROM nvcr.io/nvidia/pytorch:24.08-py3 + +# Define environments +ENV MAX_JOBS=32 +ENV VLLM_WORKER_MULTIPROC_METHOD=spawn +ENV DEBIAN_FRONTEND=noninteractive +ENV NODE_OPTIONS="" +ENV PIP_ROOT_USER_ACTION=ignore +ENV HF_HUB_ENABLE_HF_TRANSFER="1" + +# Define installation arguments +ARG APT_SOURCE=https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ +ARG PIP_INDEX=https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple +ARG VLLM_COMMIT=227578480d71fc94ef46ca77fb69496412158d68 + +# Set apt source +RUN cp /etc/apt/sources.list /etc/apt/sources.list.bak && \ + { \ + echo "deb ${APT_SOURCE} jammy main restricted universe multiverse"; \ + echo "deb ${APT_SOURCE} jammy-updates main restricted universe multiverse"; \ + echo "deb ${APT_SOURCE} jammy-backports main restricted universe multiverse"; \ + echo "deb ${APT_SOURCE} jammy-security main restricted universe multiverse"; \ + } > /etc/apt/sources.list + +# Install systemctl +RUN apt-get update && \ + apt-get install -y -o Dpkg::Options::="--force-confdef" systemd && \ + apt-get clean + +# Install tini +RUN apt-get update && \ + apt-get install -y tini && \ + apt-get clean + +# Change pip source +RUN pip config set global.index-url "${PIP_INDEX}" && \ + pip config set global.extra-index-url "${PIP_INDEX}" && \ + python -m pip install --upgrade pip + +# Uninstall nv-pytorch fork +RUN pip uninstall -y torch torchvision torchaudio \ + pytorch-quantization pytorch-triton torch-tensorrt \ + transformer_engine flash_attn apex megatron-core \ + xgboost opencv grpcio + +# Fix cv2 +RUN rm -rf /usr/local/lib/python3.10/dist-packages/cv2 + +# Install vllm-0.7.4-nightly +RUN pip install --no-cache-dir vllm --pre --extra-index-url "https://wheels.vllm.ai/${VLLM_COMMIT}" && \ + git clone -b verl_v1 https://github.com/hiyouga/vllm.git && \ + cp -r vllm/vllm/ /usr/local/lib/python3.10/dist-packages/ + +# Install torch-2.5.1 +RUN pip install --no-cache-dir "torch==2.5.1" "torchvision==0.20.1" "torchaudio==2.5.1" tensordict torchdata \ + "transformers>=4.49.0" accelerate datasets peft hf-transfer \ + ray[default] codetiming hydra-core pandas "pyarrow>=15.0.0" pylatexenc qwen-vl-utils wandb liger-kernel mathruler \ + pytest yapf py-spy pyext pre-commit ruff + +# Install flash_attn-2.7.4.post1 +RUN wget -nv https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.5cxx11abiFALSE-cp310-cp310-linux_x86_64.whl && \ + pip install --no-cache-dir flash_attn-2.7.4.post1+cu12torch2.5cxx11abiFALSE-cp310-cp310-linux_x86_64.whl + +# Fix packages +RUN pip uninstall -y pynvml nvidia-ml-py && \ + pip install --no-cache-dir --upgrade "nvidia-ml-py>=12.560.30" "fastapi[standard]>=0.115.0" "optree>=0.13.0" "pydantic>=2.9" "grpcio>=1.62.1" + +# Reset pip config +RUN pip config unset global.index-url && \ + pip config unset global.extra-index-url diff --git a/EasyR1/LICENSE b/EasyR1/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..261eeb9e9f8b2b4b0d119366dda99c6fd7d35c64 --- /dev/null +++ b/EasyR1/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/EasyR1/Makefile b/EasyR1/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..ee0caff325f202b5509919f907bcc9bd4bfcd2b9 --- /dev/null +++ b/EasyR1/Makefile @@ -0,0 +1,24 @@ +.PHONY: build commit license quality style test + +check_dirs := examples scripts tests verl setup.py + +build: + python3 setup.py sdist bdist_wheel + +commit: + pre-commit install + pre-commit run --all-files + +license: + python3 tests/check_license.py $(check_dirs) + +quality: + ruff check $(check_dirs) + ruff format --check $(check_dirs) + +style: + ruff check $(check_dirs) --fix + ruff format $(check_dirs) + +test: + pytest -vv tests/ diff --git a/EasyR1/README.md b/EasyR1/README.md new file mode 100644 index 0000000000000000000000000000000000000000..d2e6a0e37b19d819807b3bfa248e177f321584c2 --- /dev/null +++ b/EasyR1/README.md @@ -0,0 +1,218 @@ +# EasyR1: An Efficient, Scalable, Multi-Modality RL Training Framework + +[![GitHub Repo stars](https://img.shields.io/github/stars/hiyouga/EasyR1)](https://github.com/hiyouga/EasyR1/stargazers) +[![Twitter](https://img.shields.io/twitter/follow/llamafactory_ai)](https://twitter.com/llamafactory_ai) + +This project is a clean fork of the original [veRL](https://github.com/volcengine/verl) project to support vision language models, we thank all the authors for providing such a high-performance RL training framework. + +EasyR1 is efficient and scalable due to the design of **[HybirdEngine](https://arxiv.org/abs/2409.19256)** and the latest release of **[vLLM](https://github.com/vllm-project/vllm)**'s SPMD mode. + +## Features + +- Supported models + - Llama3/Qwen2/Qwen2.5/Qwen3 language models + - Qwen2/Qwen2.5-VL vision language models + - DeepSeek-R1 distill models + +- Supported algorithms + - GRPO + - Reinforce++ + - ReMax + - RLOO + +- Supported datasets + - Any text, vision-text dataset in a [specific format](#custom-dataset) + +- Supported tricks + - Padding-free training + - Resuming from checkpoint + - Wandb & SwanLab & Mlflow & Tensorboard tracking + +## Requirements + +### Software Requirements + +- Python 3.9+ +- transformers>=4.51.0 +- flash-attn>=2.4.3 +- vllm>=0.8.3 + +We provide a [Dockerfile](./Dockerfile) to easily build environments. + +We recommend using the [pre-built docker image](https://hub.docker.com/r/hiyouga/verl) in EasyR1. + +```bash +docker pull hiyouga/verl:ngc-th2.6.0-cu126-vllm0.8.4-flashinfer0.2.2-cxx11abi0 +``` + +### Hardware Requirements + +\* *estimated* + +| Method | Bits | 1.5B | 3B | 7B | 32B | +| ------------------------ | ---- | ------ | ------ | ------ | ------- | +| GRPO Full Fine-Tuning | AMP | 2*24GB | 4*40GB | 8*40GB | 16*80GB | +| GRPO Full Fine-Tuning | BF16 | 1*24GB | 1*40GB | 4*40GB | 8*80GB | + +> [!NOTE] +> Use `worker.actor.fsdp.torch_dtype=bf16` and `worker.actor.optim.strategy=adamw_bf16` to enable bf16 training. +> +> We are working hard to reduce the VRAM in RL training, LoRA support will be integrated in next updates. + +## Tutorial: Run Qwen2.5-VL GRPO on [Geometry3K](https://huggingface.co/datasets/hiyouga/geometry3k) Dataset in Just 3 Steps + +![image](assets/qwen2_5_vl_7b_geo.png) + +### Installation + +```bash +git clone https://github.com/hiyouga/EasyR1.git +cd EasyR1 +pip install -e . +``` + +### GRPO Training + +```bash +bash examples/qwen2_5_vl_7b_geo3k_grpo.sh +``` + +### Merge Checkpoint in Hugging Face Format + +```bash +python3 scripts/model_merger.py --local_dir checkpoints/easy_r1/exp_name/global_step_1/actor +``` + +> [!TIP] +> If you encounter issues with connecting to Hugging Face, consider using `export HF_ENDPOINT=https://hf-mirror.com`. +> +> If you want to use SwanLab logger, consider using `bash examples/qwen2_5_vl_7b_geo3k_swanlab.sh`. + +## Custom Dataset + +Please refer to the example datasets to prepare your own dataset. + +- Text dataset: https://huggingface.co/datasets/hiyouga/math12k +- Image-text dataset: https://huggingface.co/datasets/hiyouga/geometry3k +- Multi-image-text dataset: https://huggingface.co/datasets/hiyouga/journeybench-multi-image-vqa + +## How to Understand GRPO in EasyR1 + +![image](assets/easyr1_grpo.png) + +- To learn about the GRPO algorithm, you can refer to [Hugging Face's blog](https://huggingface.co/docs/trl/v0.16.1/en/grpo_trainer). + +## How to Run 70B+ Model in Multi-node Environment + +1. Start the Ray head node. + +```bash +ray start --head --port=6379 --dashboard-host=0.0.0.0 +``` + +2. Start the Ray worker node and connect to the head node. + +```bash +ray start --address=:6379 +``` + +3. Check the Ray resource pool. + +```bash +ray status +``` + +4. Run training script on the Ray head node only. + +```bash +bash examples/qwen2_5_vl_7b_geo3k_grpo.sh +``` + +See the **[veRL's official doc](https://verl.readthedocs.io/en/latest/start/multinode.html)** for more details about multi-node training and Ray debugger. + +## Other Baselines + +We also reproduced the following two baselines of the [R1-V](https://github.com/deep-agent/R1-V) project. +- [CLEVR-70k-Counting](examples/baselines/qwen2_5_vl_3b_clevr.sh): Train the Qwen2.5-VL-3B-Instruct model on counting problem. +- [GeoQA-8k](examples/baselines/qwen2_5_vl_3b_geoqa8k.sh): Train the Qwen2.5-VL-3B-Instruct model on GeoQA problem. + +## Performance Baselines + +See [baselines.md](assets/baselines.md). + +## Awesome Work using EasyR1 + +- **MMR1**: Advancing the Frontiers of Multimodal Reasoning. [![[code]](https://img.shields.io/github/stars/LengSicong/MMR1)](https://github.com/LengSicong/MMR1) +- **Vision-R1**: Incentivizing Reasoning Capability in Multimodal Large Language Models. [![[code]](https://img.shields.io/github/stars/Osilly/Vision-R1)](https://github.com/Osilly/Vision-R1) [![[arxiv]](https://img.shields.io/badge/arxiv-2503.06749-blue)](https://arxiv.org/abs/2503.06749) +- **Seg-Zero**: Reasoning-Chain Guided Segmentation via Cognitive Reinforcement. [![[code]](https://img.shields.io/github/stars/dvlab-research/Seg-Zero)](https://github.com/dvlab-research/Seg-Zero) [![[arxiv]](https://img.shields.io/badge/arxiv-2503.06520-blue)](https://arxiv.org/abs/2503.06520) +- **MetaSpatial**: Reinforcing 3D Spatial Reasoning in VLMs for the Metaverse. [![[code]](https://img.shields.io/github/stars/PzySeere/MetaSpatial)](https://github.com/PzySeere/MetaSpatial) [![[arxiv]](https://img.shields.io/badge/arxiv-2503.18470-blue)](https://arxiv.org/abs/2503.18470) +- **Temporal-R1**: Envolving Temporal Reasoning Capability into LMMs via Temporal Consistent Reward. [![[code]](https://img.shields.io/github/stars/appletea233/Temporal-R1)](https://github.com/appletea233/Temporal-R1) +- **NoisyRollout**: Reinforcing Visual Reasoning with Data Augmentation. [![[code]](https://img.shields.io/github/stars/John-AI-Lab/NoisyRollout)](https://github.com/John-AI-Lab/NoisyRollout) [![[arxiv]](https://img.shields.io/badge/arxiv-2504.13055-blue)](https://arxiv.org/pdf/2504.13055) +- **GUI-R1**: A Generalist R1-Style Vision-Language Action Model For GUI Agents. [![[code]](https://img.shields.io/github/stars/ritzz-ai/GUI-R1)](https://github.com/ritzz-ai/GUI-R1) [![[arxiv]](https://img.shields.io/badge/arxiv-2504.10458-blue)](https://arxiv.org/abs/2504.10458) +- **R1-Track**: Direct Application of MLLMs to Visual Object Tracking via Reinforcement Learning. [![[code]](https://img.shields.io/github/stars/Wangbiao2/R1-Track)](https://github.com/Wangbiao2/R1-Track) +- **VisionReasoner**: Unified Visual Perception and Reasoning via Reinforcement Learning. [![[code]](https://img.shields.io/github/stars/dvlab-research/VisionReasoner)](https://github.com/dvlab-research/VisionReasoner) [![[arxiv]](https://img.shields.io/badge/arxiv-2505.12081-blue)](https://arxiv.org/abs/2505.12081) +- **MM-UPT**: Unsupervised Post-Training for Multi-Modal LLM Reasoning via GRPO. [![[code]](https://img.shields.io/github/stars/waltonfuture/MM-UPT)](https://github.com/waltonfuture/MM-UPT) [![[arxiv]](https://img.shields.io/badge/arxiv-2505.22453-blue)](https://arxiv.org/pdf/2505.22453) +- **RL-with-Cold-Start**: Advancing Multimodal Reasoning via Reinforcement Learning with Cold Start. [![[code]](https://img.shields.io/github/stars/waltonfuture/RL-with-Cold-Start)](https://github.com/waltonfuture/RL-with-Cold-Start) [![[arxiv]](https://img.shields.io/badge/arxiv-2505.22334-blue)](https://arxiv.org/pdf/2505.22334) +- **ViGoRL**: Grounded Reinforcement Learning for Visual Reasoning. [![[code]](https://img.shields.io/github/stars/Gabesarch/grounded-rl)](https://github.com/Gabesarch/grounded-rl) [![[arxiv]](https://img.shields.io/badge/arxiv-2505.22334-blue)](https://arxiv.org/abs/2505.23678) +- **Revisual-R1**: Advancing Multimodal Reasoning: From Optimized Cold Start to Staged Reinforcement Learning. [![[code]](https://img.shields.io/github/stars/CSfufu/Revisual-R1)](https://github.com/CSfufu/Revisual-R1) [![[arxiv]](https://img.shields.io/badge/arxiv-2506.04207-blue)](https://arxiv.org/abs/2506.04207) +- **SophiaVL-R1**: Reinforcing MLLMs Reasoning with Thinking Reward. [![[code]](https://img.shields.io/github/stars/kxfan2002/SophiaVL-R1)](https://github.com/kxfan2002/SophiaVL-R1) [![[arxiv]](https://img.shields.io/badge/arxiv-2505.17018-blue)](https://arxiv.org/abs/2505.17018) +- **Vision-Matters**: Simple Visual Perturbations Can Boost Multimodal Math Reasoning. [![[code]](https://img.shields.io/github/stars/YutingLi0606/Vision-Matters)](https://github.com/YutingLi0606/Vision-Matters) [![[arxiv]](https://img.shields.io/badge/arxiv-2506.09736-blue)](https://arxiv.org/abs/2506.09736) + +## TODO + +- Support LoRA (high priority). +- Support ulysses parallelism for VLMs (middle priority). +- Support more VLM architectures. + +> [!NOTE] +> We will not provide scripts for supervised fine-tuning and inference in this project. If you have such requirements, we recommend using [LLaMA-Factory](https://github.com/hiyouga/LLaMA-Factory). + +### Known bugs + +These features are temporarily disabled for now, we plan to fix them one-by-one in the future updates. + +- Vision language models are not compatible with ulysses parallelism yet. + +## Discussion Group + +👋 Join our [WeChat group](assets/wechat.jpg). + +## FAQs + +> ValueError: Image features and image tokens do not match: tokens: 8192, features 9800 + +Increase the `data.max_prompt_length` or reduce the `data.max_pixels`. + +> RuntimeError: CUDA Error: out of memory at /workspace/csrc/cumem_allocator.cpp:62 + +Reduce the `worker.rollout.gpu_memory_utilization` and enable `worker.actor.offload.offload_params`. + +> RuntimeError: 0 active drivers ([]). There should only be one. + +Uninstall `deepspeed` from the current python environment. + +## Citation + +Core contributors: [Yaowei Zheng](https://github.com/hiyouga), [Junting Lu](https://github.com/AL-377), [Shenzhi Wang](https://github.com/Shenzhi-Wang), [Zhangchi Feng](https://github.com/BUAADreamer), [Dongdong Kuang](https://github.com/Kuangdd01) and Yuwen Xiong + +We also thank Guangming Sheng and Chi Zhang for helpful discussions. + +```bibtex +@misc{zheng2025easyr1, + title = {EasyR1: An Efficient, Scalable, Multi-Modality RL Training Framework}, + author = {Yaowei Zheng, Junting Lu, Shenzhi Wang, Zhangchi Feng, Dongdong Kuang, Yuwen Xiong}, + howpublished = {\url{https://github.com/hiyouga/EasyR1}}, + year = {2025} +} +``` + +We recommend to also cite the original work. + +```bibtex +@article{sheng2024hybridflow, + title = {HybridFlow: A Flexible and Efficient RLHF Framework}, + author = {Guangming Sheng and Chi Zhang and Zilingfeng Ye and Xibin Wu and Wang Zhang and Ru Zhang and Yanghua Peng and Haibin Lin and Chuan Wu}, + year = {2024}, + journal = {arXiv preprint arXiv: 2409.19256} +} +``` diff --git a/EasyR1/assets/baselines.md b/EasyR1/assets/baselines.md new file mode 100644 index 0000000000000000000000000000000000000000..4f1d1fa126e7081c61cdaed3b7b67b4690d6c911 --- /dev/null +++ b/EasyR1/assets/baselines.md @@ -0,0 +1,53 @@ +# Baselines + +Environment: [hiyouga/verl:ngc-th2.6.0-cu126-vllm0.8.3-flashinfer0.2.2-cxx11abi0](https://hub.docker.com/layers/hiyouga/verl/ngc-th2.6.0-cu126-vllm0.8.3-flashinfer0.2.2-cxx11abi0/images/sha256-335ed6cd1fe73090e458409cfa4394d6abf4cd0503ca44dbafdc28ff72e5ed20) + +EasyR1 version: [v0.3.0](https://github.com/hiyouga/EasyR1/tree/v0.3.0) + +Welcome to contribute new data points! + +## Algorithm Baselines + +### [Qwen2.5-Instruct](https://huggingface.co/Qwen/Qwen2.5-7B-Instruct) on [Math12k](https://huggingface.co/datasets/hiyouga/math12k) + +| Size | Algorithm | Bits | LR | KL | Test Score | +| ---- | ----------- | ---- | ---- | ---- | ---------- | +| 7B | GRPO | AMP | 1e-6 | 1e-2 | 0.73->0.79 | + +### [Qwen2.5-VL-Instruct](https://huggingface.co/Qwen/Qwen2.5-VL-7B-Instruct) on [Geometry3k](https://huggingface.co/datasets/hiyouga/geometry3k) + +| Size | Algorithm | Bits | LR | KL | Test Score | +| ---- | ----------- | ---- | ---- | ---- | ---------- | +| 7B | GRPO | AMP | 1e-6 | 1e-2 | 0.39->0.52 | +| 7B | GRPO | BF16 | 1e-6 | 1e-2 | 0.39->0.52 | +| 7B | GRPO | AMP | 1e-6 | 1e-3 | 0.39->0.52 | +| 7B | RLOO | AMP | 1e-6 | 1e-2 | 0.39->0.53 | +| 3B | GRPO | AMP | 1e-6 | 1e-2 | 0.27->0.44 | +| 32B | GRPO | BF16 | 1e-6 | 1e-2 | 0.46->0.61 | + +> [!NOTE] +> The hyper-parameters not listed are all the same as the default values. + +## Performance Baselines + +### [Qwen2.5-VL-Instruct](https://huggingface.co/Qwen/Qwen2.5-VL-7B-Instruct) on [Geometry3k](https://huggingface.co/datasets/hiyouga/geometry3k) + +| Size | GPU Type | Bits | Batch Size | vLLM Util | vLLM TP | Peak Mem | Peak VRAM | Throughput | Sec per step | Actor MFU | +| ---- | ------------- | ---- | ---------- | --------- | ------- | -------- | --------- | ---------- | ------------ | --------- | +| 3B | 8 * H100 80GB | AMP | 4 / 16 | 0.6 | 2 | 120GB | 35GB | 1200 | 180s | 6.3% | +| 7B | 8 * H100 80GB | AMP | 4 / 16 | 0.6 | 2 | 140GB | 60GB | 1200 | 180s | 13.6% | +| 7B | 8 * H100 80GB | AMP | 10 / 20 | 0.6 | 2 | 150GB | 75GB | 1400 | 170s | 19.2% | +| 7B | 8 * L20 48GB | AMP | 4 / 16 | 0.6 | 2 | 150GB | 44GB | 410 | 580s | 26.5% | +| 7B | 8 * H100 80GB | BF16 | 4 / 16 | 0.6 | 2 | 150GB | 50GB | 1280 | 190s | 13.9% | +| 32B | 8 * H100 80GB | BF16 | 1 / 8 | 0.6 | 8 | 240GB | 68GB | 360 | 860s | 11.2% | + +- Batch Size: micro_batch_size_per_device_for_update / micro_batch_size_per_device_for_experience +- vLLM Util: rollout.gpu_memory_utilization +- vLLM TP: rollout.tensor_parallel_size +- Peak Mem: Peak CPU memory usage +- Peak VRAM: Peak GPU memory usage +- Throughput: Number of tokens per second per GPU by one training step +- Sec per step: Average time per step in seconds + +> [!NOTE] +> The hyper-parameters not listed are all the same as the default values. diff --git a/EasyR1/assets/qwen2_5_vl_7b_geo.png b/EasyR1/assets/qwen2_5_vl_7b_geo.png new file mode 100644 index 0000000000000000000000000000000000000000..a61f5fcc6cb9e287e17a84e8236f643b006699cd Binary files /dev/null and b/EasyR1/assets/qwen2_5_vl_7b_geo.png differ diff --git a/EasyR1/examples/baselines/qwen2_5_vl_3b_clevr.sh b/EasyR1/examples/baselines/qwen2_5_vl_3b_clevr.sh new file mode 100644 index 0000000000000000000000000000000000000000..b3730ebd20b5dfd8c485f8e9aa1085d3e3df8ecc --- /dev/null +++ b/EasyR1/examples/baselines/qwen2_5_vl_3b_clevr.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -x + +export PYTHONUNBUFFERED=1 + +MODEL_PATH=Qwen/Qwen2.5-VL-3B-Instruct # replace it with your local file path + +python3 -m verl.trainer.main \ + config=examples/config.yaml \ + data.train_files=BUAADreamer/clevr_count_70k@train \ + data.val_files=BUAADreamer/clevr_count_70k@test \ + data.format_prompt=./examples/format_prompt/r1v_format.jinja \ + worker.actor.model.model_path=${MODEL_PATH} \ + worker.rollout.tensor_parallel_size=1 \ + worker.reward.reward_type=sequential \ + worker.reward.reward_function=./examples/reward_function/r1v.py:compute_score \ + trainer.experiment_name=qwen2_5_vl_3b_clevr \ + trainer.n_gpus_per_node=2 diff --git a/EasyR1/examples/baselines/qwen2_5_vl_3b_geoqa8k.sh b/EasyR1/examples/baselines/qwen2_5_vl_3b_geoqa8k.sh new file mode 100644 index 0000000000000000000000000000000000000000..c19c4fc477dbcf4faaa441c6ae9630bc59a0bcf8 --- /dev/null +++ b/EasyR1/examples/baselines/qwen2_5_vl_3b_geoqa8k.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -x + +export PYTHONUNBUFFERED=1 + +MODEL_PATH=Qwen/Qwen2.5-VL-3B-Instruct # replace it with your local file path + +python3 -m verl.trainer.main \ + config=examples/config.yaml \ + data.train_files=leonardPKU/GEOQA_8K_R1V@train \ + data.val_files=leonardPKU/GEOQA_8K_R1V@test \ + data.format_prompt=./examples/format_prompt/r1v_format.jinja \ + worker.actor.model.model_path=${MODEL_PATH} \ + worker.rollout.tensor_parallel_size=1 \ + worker.reward.reward_type=sequential \ + worker.reward.reward_function=./examples/reward_function/r1v.py:compute_score \ + trainer.experiment_name=qwen2_5_vl_3b_geoqa8k \ + trainer.n_gpus_per_node=8 diff --git a/EasyR1/examples/config.yaml b/EasyR1/examples/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9517c9c5fe1e876d81ec29a1118af51a411fad80 --- /dev/null +++ b/EasyR1/examples/config.yaml @@ -0,0 +1,94 @@ +data: + train_files: hiyouga/math12k@train + val_files: hiyouga/math12k@test + prompt_key: question + answer_key: answer + image_key: images + image_dir: null + max_prompt_length: 8192 + max_response_length: 16384 + rollout_batch_size: 128 + val_batch_size: 256 + format_prompt: /nas/shared/kilab/wangyujia/EasyR1/examples/format_prompt/bio_format.jinja + override_chat_template: null + shuffle: true + seed: 1 + min_pixels: 262144 + max_pixels: 4194304 + filter_overlong_prompts: true + +algorithm: + adv_estimator: grpo + disable_kl: false + use_kl_loss: true + kl_penalty: low_var_kl + kl_coef: 1.0e-2 + +worker: + actor: + global_batch_size: 64 + micro_batch_size_per_device_for_update: 2 #跟gpu显存相关 + micro_batch_size_per_device_for_experience: 16 + max_grad_norm: 1.0 + padding_free: true + ulysses_sequence_parallel_size: 1 + model: + model_path: Qwen/Qwen2.5-7B-Instruct + enable_gradient_checkpointing: true + trust_remote_code: false + freeze_vision_tower: false + optim: + lr: 1.0e-6 + weight_decay: 1.0e-2 + strategy: adamw # {adamw, adamw_bf16} + lr_warmup_ratio: 0.0 + fsdp: + enable_full_shard: true + enable_cpu_offload: false + enable_rank0_init: true + offload: + offload_params: true # true: more CPU memory; false: more GPU memory + offload_optimizer: true # true: more CPU memory; false: more GPU memory + + rollout: + n: 5 + temperature: 1.0 + top_p: 0.99 + gpu_memory_utilization: 0.6 + enforce_eager: false + enable_chunked_prefill: false + tensor_parallel_size: 2 + limit_images: 0 + max_num_batched_tokens: 24576 + val_override_config: + temperature: 0.5 + n: 1 + + ref: + fsdp: + enable_full_shard: true + enable_cpu_offload: true # true: more CPU memory; false: more GPU memory + enable_rank0_init: true + offload: + offload_params: false + + reward: + reward_type: batch + reward_function: /nas/shared/kilab/wangyujia/EasyR1/examples/reward_function/bio.py:compute_score + +trainer: + total_epochs: 15 + max_steps: null + project_name: easy_r1 + experiment_name: qwen2_5_7b_math_grpo + logger: ["console", "wandb"] + nnodes: 1 + n_gpus_per_node: 8 + val_freq: 5 # -1 to disable + val_before_train: true + val_only: false + val_generations_to_log: 3 + save_freq: 5 # -1 to disable + save_limit: 3 # -1 to disable + save_checkpoint_path: /oss/wangyujia/BIO/rl/qwen2.5_7b_bio_06162150 + load_checkpoint_path: null diff --git a/EasyR1/examples/format_prompt/bio_format.jinja b/EasyR1/examples/format_prompt/bio_format.jinja new file mode 100644 index 0000000000000000000000000000000000000000..7cf6628dc7b22d6f9116c0e0f244597f871a3882 --- /dev/null +++ b/EasyR1/examples/format_prompt/bio_format.jinja @@ -0,0 +1,2 @@ +{{ content | trim }} You must first reason through the question step by step, as if you're thinking aloud. Enclose your full reasoning process within tags. After your reasoning, output only the number corresponding to the final answer choice inside tags.For example: reasoning process result number + diff --git a/EasyR1/examples/format_prompt/math_format.jinja b/EasyR1/examples/format_prompt/math_format.jinja new file mode 100644 index 0000000000000000000000000000000000000000..8d6aa2344c57d654c87dbd26d90f7af473f82cc6 --- /dev/null +++ b/EasyR1/examples/format_prompt/math_format.jinja @@ -0,0 +1 @@ +{{ content | trim }} You FIRST think about the reasoning process as an internal monologue and then provide the final answer. The reasoning process MUST BE enclosed within tags. The final answer MUST BE put in \boxed{}. diff --git a/EasyR1/examples/format_prompt/r1v_format.jinja b/EasyR1/examples/format_prompt/r1v_format.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ecf6f471146202113d4e44c10777d60e65c8e8a --- /dev/null +++ b/EasyR1/examples/format_prompt/r1v_format.jinja @@ -0,0 +1 @@ +{{ content | trim }} A conversation between User and Assistant. The user asks a question, and the Assistant solves it. The assistant first thinks about the reasoning process in the mind and then provides the user with the answer. The reasoning process and answer are enclosed within and tags, respectively, i.e., reasoning process here answer here diff --git a/EasyR1/examples/multi_qwen2.5_bio.sh b/EasyR1/examples/multi_qwen2.5_bio.sh new file mode 100644 index 0000000000000000000000000000000000000000..4ee55a79320b14a62869a5b9af6e2895531a3f85 --- /dev/null +++ b/EasyR1/examples/multi_qwen2.5_bio.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +set -x + +export PYTHONUNBUFFERED=1 +export master_addr=$MASTER_ADDR +export master_port=$MASTER_PORT +export nnodes=$WORLD_SIZE +export node_rank=$RANK +export nproc_per_node=$NPROC_PER_NODE +export WANDB_BASE_URL=https://api.bandw.top + +MODEL_PATH=/oss/wangyujia/BIO/pretrain_output/qwen2.5-7b-instruct-bio/bio_all/save1epoch/checkpoint-1300 # replace it with your local file path + +python3 -m verl.trainer.main \ + config=config.yaml \ + data.train_files=/nas/shared/kilab/wangyujia/rl_data@train \ + data.val_files=/nas/shared/kilab/wangyujia/rl_data@validation \ + data.max_prompt_length=8192 \ + data.max_response_length=8192\ + data.rollout_batch_size=256 \ + worker.actor.model.model_path=${MODEL_PATH} \ + worker.rollout.tensor_parallel_size=1 \ + worker.rollout.n=8 \ + trainer.experiment_name=qwen2_5_bio_grpo \ + trainer.n_gpus_per_node=8 \ + trainer.total_epochs=5 \ + trainer.save_checkpoint_path=/oss/wangyujia/BIO/rl/qwen2.5_7b_bio diff --git a/EasyR1/examples/qwen2.5_bio.sh b/EasyR1/examples/qwen2.5_bio.sh new file mode 100644 index 0000000000000000000000000000000000000000..acc30403f881b87a7304af624a39bf124bc64ae4 --- /dev/null +++ b/EasyR1/examples/qwen2.5_bio.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -x + +export PYTHONUNBUFFERED=1 +export WANDB_BASE_URL=https://api.bandw.top + +MODEL_PATH=/oss/wangyujia/BIO/pretrain_output/qwen2.5-7b-instruct-bio/bio_all/save1epoch/checkpoint-1300 # replace it with your local file path + +python3 -m verl.trainer.main \ + config=config.yaml \ + data.train_files=/nas/shared/kilab/wangyujia/rl_data/deeplocmulti@train \ + data.val_files=/nas/shared/kilab/wangyujia/rl_data/deeplocmulti@validation \ + data.max_prompt_length=4096 \ + data.max_response_length=16384\ + worker.actor.model.model_path=${MODEL_PATH} \ + worker.rollout.tensor_parallel_size=1 \ + worker.rollout.n=5 \ + trainer.experiment_name=qwen2.5_7b_bio_06182042 \ + trainer.n_gpus_per_node=8 \ + trainer.total_epochs=1 \ + trainer.save_checkpoint_path=/oss/wangyujia/BIO/rl/qwen2.5_7b_bio_06182042 \ No newline at end of file diff --git a/EasyR1/examples/qwen2_5_7b_math_grpo.sh b/EasyR1/examples/qwen2_5_7b_math_grpo.sh new file mode 100644 index 0000000000000000000000000000000000000000..842afa5c149624b8e4aadca8aec061042a5d7014 --- /dev/null +++ b/EasyR1/examples/qwen2_5_7b_math_grpo.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -x + +export PYTHONUNBUFFERED=1 + +MODEL_PATH=Qwen/Qwen2.5-7B-Instruct # replace it with your local file path + +python3 -m verl.trainer.main \ + config=examples/config.yaml \ + worker.actor.model.model_path=${MODEL_PATH} diff --git a/EasyR1/examples/qwen2_5_vl_32b_geo3k_grpo.sh b/EasyR1/examples/qwen2_5_vl_32b_geo3k_grpo.sh new file mode 100644 index 0000000000000000000000000000000000000000..6b9e8f2e5475559d41842d961b9997876463f096 --- /dev/null +++ b/EasyR1/examples/qwen2_5_vl_32b_geo3k_grpo.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -x + +export PYTHONUNBUFFERED=1 + +MODEL_PATH=Qwen/Qwen2.5-VL-32B-Instruct # replace it with your local file path + +python3 -m verl.trainer.main \ + config=examples/config.yaml \ + data.train_files=hiyouga/geometry3k@train \ + data.val_files=hiyouga/geometry3k@test \ + worker.actor.model.model_path=${MODEL_PATH} \ + worker.actor.micro_batch_size_per_device_for_update=1 \ + worker.actor.micro_batch_size_per_device_for_experience=8 \ + worker.actor.fsdp.torch_dtype=bf16 \ + worker.actor.optim.strategy=adamw_bf16 \ + worker.rollout.tensor_parallel_size=8 \ + trainer.experiment_name=qwen2_5_vl_32b_geo_grpo \ + trainer.n_gpus_per_node=8 diff --git a/EasyR1/examples/qwen2_5_vl_3b_geo3k_grpo.sh b/EasyR1/examples/qwen2_5_vl_3b_geo3k_grpo.sh new file mode 100644 index 0000000000000000000000000000000000000000..d783ad3f112740b3caa28bca8f04fd57d0d41732 --- /dev/null +++ b/EasyR1/examples/qwen2_5_vl_3b_geo3k_grpo.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -x + +export PYTHONUNBUFFERED=1 + +MODEL_PATH=Qwen/Qwen2.5-VL-3B-Instruct # replace it with your local file path + +python3 -m verl.trainer.main \ + config=examples/config.yaml \ + data.train_files=hiyouga/geometry3k@train \ + data.val_files=hiyouga/geometry3k@test \ + worker.actor.model.model_path=${MODEL_PATH} \ + worker.rollout.tensor_parallel_size=1 \ + trainer.experiment_name=qwen2_5_vl_3b_geo_grpo \ + trainer.n_gpus_per_node=2 diff --git a/EasyR1/examples/qwen2_5_vl_7b_geo3k_grpo.sh b/EasyR1/examples/qwen2_5_vl_7b_geo3k_grpo.sh new file mode 100644 index 0000000000000000000000000000000000000000..705278420008534f3449c4d4222fcc47e83fb333 --- /dev/null +++ b/EasyR1/examples/qwen2_5_vl_7b_geo3k_grpo.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -x + +export PYTHONUNBUFFERED=1 + +MODEL_PATH=Qwen/Qwen2.5-VL-7B-Instruct # replace it with your local file path + +python3 -m verl.trainer.main \ + config=examples/config.yaml \ + data.train_files=hiyouga/geometry3k@train \ + data.val_files=hiyouga/geometry3k@test \ + worker.actor.model.model_path=${MODEL_PATH} \ + trainer.experiment_name=qwen2_5_vl_7b_geo_grpo \ + trainer.n_gpus_per_node=8 diff --git a/EasyR1/examples/qwen2_5_vl_7b_geo3k_reinforce.sh b/EasyR1/examples/qwen2_5_vl_7b_geo3k_reinforce.sh new file mode 100644 index 0000000000000000000000000000000000000000..f44584dcc9afeb71c7a49af5409673ca7d66a4cd --- /dev/null +++ b/EasyR1/examples/qwen2_5_vl_7b_geo3k_reinforce.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -x + +export PYTHONUNBUFFERED=1 + +MODEL_PATH=Qwen/Qwen2.5-VL-7B-Instruct # replace it with your local file path + +python3 -m verl.trainer.main \ + config=examples/config.yaml \ + data.train_files=hiyouga/geometry3k@train \ + data.val_files=hiyouga/geometry3k@test \ + worker.actor.model.model_path=${MODEL_PATH} \ + algorithm.adv_estimator=reinforce_plus_plus \ + algorithm.use_kl_loss=false \ + algorithm.kl_penalty=kl \ + algorithm.kl_coef=1.0e-3 \ + trainer.experiment_name=qwen2_5_vl_7b_geo_reinforce_pp \ + trainer.n_gpus_per_node=8 diff --git a/EasyR1/examples/qwen2_5_vl_7b_geo3k_swanlab.sh b/EasyR1/examples/qwen2_5_vl_7b_geo3k_swanlab.sh new file mode 100644 index 0000000000000000000000000000000000000000..bac9e850b970e2607de8e63e4557b0151f67db0e --- /dev/null +++ b/EasyR1/examples/qwen2_5_vl_7b_geo3k_swanlab.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -x + +export PYTHONUNBUFFERED=1 + +MODEL_PATH=Qwen/Qwen2.5-VL-7B-Instruct # replace it with your local file path + +python3 -m verl.trainer.main \ + config=examples/config.yaml \ + data.train_files=hiyouga/geometry3k@train \ + data.val_files=hiyouga/geometry3k@test \ + worker.actor.model.model_path=${MODEL_PATH} \ + trainer.experiment_name=qwen2_5_vl_7b_geo_grpo \ + trainer.logger=['console','swanlab'] \ + trainer.n_gpus_per_node=8 diff --git a/EasyR1/examples/qwen2_5_vl_7b_multi_image.sh b/EasyR1/examples/qwen2_5_vl_7b_multi_image.sh new file mode 100644 index 0000000000000000000000000000000000000000..bcf9a48616ea1897fb9e96e6a64e8d2d34672b23 --- /dev/null +++ b/EasyR1/examples/qwen2_5_vl_7b_multi_image.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# REMINDER: this script uses test data split and should ONLY be used for debugging. DO NOT use for training. + +set -x + +export PYTHONUNBUFFERED=1 + +MODEL_PATH=Qwen/Qwen2.5-VL-7B-Instruct # replace it with your local file path + +python3 -m verl.trainer.main \ + config=examples/config.yaml \ + data.train_files=hiyouga/journeybench-multi-image-vqa@train \ + data.val_files=hiyouga/journeybench-multi-image-vqa@test \ + data.rollout_batch_size=256 \ + worker.actor.model.model_path=${MODEL_PATH} \ + worker.rollout.limit_images=2 \ + trainer.experiment_name=qwen2_5_vl_7b_multi_image \ + trainer.n_gpus_per_node=8 diff --git a/EasyR1/examples/qwen3_4b_math_grpo.sh b/EasyR1/examples/qwen3_4b_math_grpo.sh new file mode 100644 index 0000000000000000000000000000000000000000..32bbaac99327c298aa2aaf13e5e5f2f61b3e2d45 --- /dev/null +++ b/EasyR1/examples/qwen3_4b_math_grpo.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +set -x + +export PYTHONUNBUFFERED=1 + +MODEL_PATH=Qwen/Qwen3-4B # replace it with your local file path + +python3 -m verl.trainer.main \ + config=examples/config.yaml \ + data.max_response_length=4096 \ + worker.actor.model.model_path=${MODEL_PATH} \ + trainer.experiment_name=qwen3_4b_math_grpo diff --git a/EasyR1/examples/reward_function/__pycache__/bio.cpython-310.pyc b/EasyR1/examples/reward_function/__pycache__/bio.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..36a1cc246464ba4a01a7faa02d7282c8d311f7c1 Binary files /dev/null and b/EasyR1/examples/reward_function/__pycache__/bio.cpython-310.pyc differ diff --git a/EasyR1/examples/reward_function/bio.py b/EasyR1/examples/reward_function/bio.py new file mode 100644 index 0000000000000000000000000000000000000000..3eed5582876e57d1ec0d0f0d092284e8a8841eaa --- /dev/null +++ b/EasyR1/examples/reward_function/bio.py @@ -0,0 +1,183 @@ + + + +from itertools import islice, zip_longest +from typing import Callable, Dict, List, Optional, Tuple, TypedDict +import json + +def repeatness_reward(s: str): + def ranks(l): + index = {v: i for i, v in enumerate(sorted(set(l)))} + return [index[v] for v in l] + + def suffixArray(s): + line = ranks(s) + n, k, ans, sa = len(s), 1, line, [0] * len(s) + while k < n - 1: + line = ranks(list(zip_longest(line, islice(line, k, None), fillvalue=-1))) + ans, k = line, k << 1 + for i, k in enumerate(ans): + sa[k] = i + return ans, sa + + def lcp(arr, suffixArr, inv_suff): + n, ans, k = len(arr), [0] * len(arr), 0 + + for i in range(n): + if inv_suff[i] == n - 1: + k = 0 + continue + + j = suffixArr[inv_suff[i] + 1] + while i + k < n and j + k < n and arr[i + k] == arr[j + k]: + k += 1 + + ans[inv_suff[i]] = k + if k > 0: + k -= 1 + + return ans + + arr = [ord(i) for i in s] + n = len(arr) + if n <= 1: + return 0 + c, sa = suffixArray(arr) + cnt = sum(lcp(arr, sa, c)) + + return 1 - cnt * 2 / (n * (n + 1)) + +import re + +def format_reward(predict_str: str) -> float: + """ + 格式奖励函数，严格要求输出格式为： + ...... + 中间不能有多余内容 + """ + pattern = r'^.*?\s*\s*([0-9])\s*$' + return 1.0 if re.fullmatch(pattern, predict_str.strip(), re.DOTALL) else 0.0 + +def acc_reward(predict_str: str, ground_truth: str) -> float: + """ + 准确率奖励函数 + 要求中内容与ground_truth完全一致（顺序、空格等） + """ + match = re.search(r'\s*([0-9])\s*', predict_str) + if not match: + return 0.0 + answer_content = match.group(1) + # print(ground_truth) + # print(answer_content) + # print(int(answer_content) == ground_truth) + # print("ground_truth 类型：", type(ground_truth)) + # print("answer_content 类型：", type(answer_content)) + # print("========") + if int(answer_content) == ground_truth: + return 1.0 + else: + return 0.0 + # return 1.0 if answer_content == ground_truth else 0.0 + # match = re.search(r'(.*?)', predict_str, re.DOTALL) + # if not match: + # return 0.0 + # answer_content = match.group(1).strip() + # return 1.0 if answer_content == ground_truth else 0.0 + +# def compute_score( solution_str: str, ground_truth: str, extra_info): +# """ +# 综合评分函数 +# """ +def compute_score(predicts: List[str], ground_truths: List[str], format_weight: float = 0.1) -> List[Dict[str, float]]: + scores = [] + save_path="/nas/shared/kilab/wangyujia/check_rl/result-06170934.jsonl" + with open(save_path, "w", encoding="utf-8") as f: + for solution_str, ground_truth in zip(predicts, ground_truths): + format_score = format_reward(solution_str) + acc_score = acc_reward(solution_str, ground_truth) + + # 提取内容 + think_match = re.search(r'(.*?)', solution_str, re.DOTALL) + think_str = think_match.group(1).strip() if think_match else "" + repeat_score = repeatness_reward(think_str) + + scores.append( + { + "overall": format_score + acc_score + repeat_score, + "format": format_score, + "accuracy": acc_score, + "repeat" : repeat_score, + } + ) + + # 写入 JSONL 文件 + f.write(json.dumps({ + "solution_str": solution_str, + "ground_truth": ground_truth, + "overall": format_score + acc_score + repeat_score, + "format": format_score, + "accuracy": acc_score, + "repeat" : repeat_score, + }, ensure_ascii=False) + "\n") + + # 加权综合评分（格式占30%，准确率占70%） + # 合成字典 + # total_score = { + # "format_score": format_score, + # "acc_score": acc_score, + # "repeat_score": repeat_score, + # "total_score": format_score + acc_score + repeat_score + # } + #total_score=format_score + acc_score + repeat_score + + return scores + + +# print(format_reward("Step-by-step logic 5 ")) +# print(format_reward("Something\nacross lines\n 0 ")) + +# print(format_reward("No tags here")) +# print(format_reward("OK12")) # 多位数字 +# print(format_reward("OKA")) # 字母不允许 +# print(format_reward("Yes ")) # 空的答案 +# print(format_reward("OK3extra")) # 多余内容 +# print(format_reward("3Reasoning")) # 标签顺序错误 + +# print(acc_reward("Step-by-step logic 5 ",'5')) +# print(acc_reward("Something\nacross lines\n 0 ",'1')) + + +# str_="\nThe protein name is P32783, the protein amino acid sequence is MSTKPEKPIWMSQEDYDRQYGSITGDESSTVSKKDSKVTANAPGDGNGSLPVLQSSSILTSKVSDLPIEAESGFKIQKRRHERYDQEERLRKQRAQKLREEQLKRHEIEMTANRSINVDQIVREHYNERTIIANRAKRNLSPIIKLRNFNNAIKYMLIDKYTKPGDVVLELGCGKGGDLRKYGAAGISQFIGIDISNASIQEAHKRYRSMRNLDYQVVLITGDCFGESLGVAVEPFPDCRFPCDIVSTQFCLHYAFETEEKARRALLNVAKSLKIGGHFFGTIPDSEFIRYKLNKFPKEVEKPSWGNSIYKVTFENNSYQKNDYEFTSPYGQMYTYWLEDAIDNVPEYVVPFETLRSLADEYGLELVSQMPFNKFFVQEIPKWIERFSPKMREGLQRSDGRYGVEGDEKEAASYFYTMFAFRKVKQYIEPESVKPN, the protein localization prediction for P32783 is Cell.membrane,M, so the location label is 4. Therefore, option 4 is the correct answer.\n\n\n4\n" +# print(format_reward(str_)) + + + +def check_rewards(jsonl_path: str) -> List[Dict[str, float]]: + results = [] + with open(jsonl_path, "r", encoding="utf-8") as f: + for line in f: + data = json.loads(line) + solution_str = data["solution_str"] + ground_truth = data["ground_truth"] + + # 重新计算三个分数 + format_score = format_reward(solution_str) + acc_score = acc_reward(solution_str, ground_truth) + think_match = re.search(r'(.*?)', solution_str, re.DOTALL) + think_str = think_match.group(1).strip() if think_match else "" + repeat_score = repeatness_reward(think_str) + + total_score = format_score + acc_score + repeat_score + + + result = { + "format": format_score, + "accuracy": acc_score, + "repeat": repeat_score, + "overall": total_score, + } + # results.append(result) + + print(json.dumps(result, indent=2, ensure_ascii=False)) + +check_rewards("/nas/shared/kilab/wangyujia/check_rl/check.jsonl") \ No newline at end of file diff --git a/EasyR1/examples/reward_function/math.py b/EasyR1/examples/reward_function/math.py new file mode 100644 index 0000000000000000000000000000000000000000..f8887532df8cd0f86967c54e355839cb969066d6 --- /dev/null +++ b/EasyR1/examples/reward_function/math.py @@ -0,0 +1,46 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from typing import Dict, List + +from mathruler.grader import extract_boxed_content, grade_answer + + +def format_reward(predict: str) -> float: + pattern = re.compile(r".*.*\\boxed\{.*\}.*", re.DOTALL) + format_match = re.fullmatch(pattern, predict) + return 1.0 if format_match else 0.0 + + +def accuracy_reward(predict: str, ground_truth: str) -> float: + answer = extract_boxed_content(predict) + return 1.0 if grade_answer(answer, ground_truth) else 0.0 + + +def compute_score(predicts: List[str], ground_truths: List[str], format_weight: float = 0.1) -> List[Dict[str, float]]: + scores = [] + for predict, ground_truth in zip(predicts, ground_truths): + predict = re.sub(r"\s*(<|>|/)\s*", r"\1", predict) # handle qwen2.5vl-32b format + format_score = format_reward(predict) + accuracy_score = accuracy_reward(predict, ground_truth) + scores.append( + { + "overall": (1 - format_weight) * accuracy_score + format_weight * format_score, + "format": format_score, + "accuracy": accuracy_score, + } + ) + + return scores diff --git a/EasyR1/examples/reward_function/r1v.py b/EasyR1/examples/reward_function/r1v.py new file mode 100644 index 0000000000000000000000000000000000000000..204762f31458f8470ee0be7468cfc9ce5b627446 --- /dev/null +++ b/EasyR1/examples/reward_function/r1v.py @@ -0,0 +1,47 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from typing import Dict + +from mathruler.grader import grade_answer + + +def format_reward(predict: str) -> float: + pattern = re.compile(r".*?\s*.*?", re.DOTALL) + format_match = re.fullmatch(pattern, predict) + return 1.0 if format_match else 0.0 + + +def accuracy_reward(predict: str, ground_truth: str) -> float: + try: + content_match = re.search(r"(.*?)", predict) + given_answer = content_match.group(1).strip() if content_match else predict.strip() + if grade_answer(given_answer, ground_truth.strip()): + return 1.0 + + except Exception: + pass + + return 0.0 + + +def compute_score(predict: str, ground_truth: str, format_weight: float = 0.5) -> Dict[str, float]: + format_score = format_reward(predict) + accuracy_score = accuracy_reward(predict, ground_truth) + return { + "overall": (1 - format_weight) * accuracy_score + format_weight * format_score, + "format": format_score, + "accuracy": accuracy_score, + } diff --git a/EasyR1/examples/runtime_env.yaml b/EasyR1/examples/runtime_env.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a4086253d4f7198f85225941c076285ab9f5321a --- /dev/null +++ b/EasyR1/examples/runtime_env.yaml @@ -0,0 +1,9 @@ +working_dir: ./ +excludes: ["/.git/"] +env_vars: + TOKENIZERS_PARALLELISM: "true" + NCCL_DEBUG: "WARN" + VLLM_LOGGING_LEVEL: "WARN" + TORCH_NCCL_AVOID_RECORD_STREAMS: "1" + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:False" + PYTHONUNBUFFERED: "1" diff --git a/EasyR1/examples/wandb/debug-internal.log b/EasyR1/examples/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..35e13e29655e458a6e6d569c053727ec806184f9 --- /dev/null +++ b/EasyR1/examples/wandb/debug-internal.log @@ -0,0 +1,40 @@ +{"time":"2025-06-18T20:58:00.646564392+08:00","level":"INFO","msg":"stream: starting","core version":"0.20.1","symlink path":"/nas/shared/kilab/wangyujia/EasyR1/examples/wandb/run-20250618_205800-gokgllcc/logs/debug-core.log"} +{"time":"2025-06-18T20:58:01.597757208+08:00","level":"INFO","msg":"stream: created new stream","id":"gokgllcc"} +{"time":"2025-06-18T20:58:01.59781863+08:00","level":"INFO","msg":"stream: started","id":"gokgllcc"} +{"time":"2025-06-18T20:58:01.597835726+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"gokgllcc"} +{"time":"2025-06-18T20:58:01.597840362+08:00","level":"INFO","msg":"handler: started","stream_id":"gokgllcc"} +{"time":"2025-06-18T20:58:01.597852506+08:00","level":"INFO","msg":"sender: started","stream_id":"gokgllcc"} +{"time":"2025-06-18T20:58:02.234767224+08:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-06-18T21:59:50.39078701+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/easy_r1/gokgllcc/file_stream\": unexpected EOF"} +{"time":"2025-06-18T22:21:34.824892314+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/easy_r1/gokgllcc/file_stream\": read tcp 10.1.4.110:46420->172.67.193.61:443: read: connection reset by peer"} +{"time":"2025-06-18T22:24:47.62203637+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/easy_r1/gokgllcc/file_stream\": read tcp 10.1.4.110:34030->172.67.193.61:443: read: connection timed out"} +{"time":"2025-06-18T22:36:36.745449662+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/easy_r1/gokgllcc/file_stream\": unexpected EOF"} +{"time":"2025-06-18T22:43:35.685651008+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/easy_r1/gokgllcc/file_stream\": unexpected EOF"} +{"time":"2025-06-18T22:49:53.926051581+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/easy_r1/gokgllcc/file_stream\": read tcp 10.1.4.110:59664->172.67.193.61:443: read: connection timed out"} +{"time":"2025-06-18T22:50:28.123387753+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/easy_r1/gokgllcc/file_stream\": unexpected EOF"} +{"time":"2025-06-18T22:54:58.56508127+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/easy_r1/gokgllcc/file_stream\": read tcp 10.1.4.110:43628->172.67.193.61:443: read: connection timed out"} +{"time":"2025-06-18T23:03:01.17247557+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/easy_r1/gokgllcc/file_stream\": read tcp 10.1.4.110:50886->172.67.193.61:443: read: connection reset by peer"} +{"time":"2025-06-18T23:04:44.248226613+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/easy_r1/gokgllcc/file_stream\": read tcp 10.1.4.110:46096->172.67.193.61:443: read: connection reset by peer"} +{"time":"2025-06-18T23:07:54.75838842+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/easy_r1/gokgllcc/file_stream\": read tcp 10.1.4.110:37806->172.67.193.61:443: read: connection timed out"} +{"time":"2025-06-18T23:10:38.086066562+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/easy_r1/gokgllcc/file_stream\": read tcp 10.1.4.110:39150->172.67.193.61:443: read: connection timed out"} +{"time":"2025-06-18T23:13:04.583303721+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/easy_r1/gokgllcc/file_stream\": read tcp 10.1.4.110:33602->172.67.193.61:443: read: connection reset by peer"} +{"time":"2025-06-18T23:16:17.029083057+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/easy_r1/gokgllcc/file_stream\": read tcp 10.1.4.110:33840->172.67.193.61:443: read: connection timed out"} +{"time":"2025-06-18T23:19:25.446088292+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/easy_r1/gokgllcc/file_stream\": read tcp 10.1.4.110:52070->172.67.193.61:443: read: connection timed out"} +{"time":"2025-06-18T23:30:41.797073652+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/easy_r1/gokgllcc/file_stream\": read tcp 10.1.4.110:37098->172.67.193.61:443: read: connection timed out"} +{"time":"2025-06-18T23:35:01.381128603+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/easy_r1/gokgllcc/file_stream\": read tcp 10.1.4.110:45100->172.67.193.61:443: read: connection timed out"} +{"time":"2025-06-18T23:42:05.833272675+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/easy_r1/gokgllcc/file_stream\": unexpected EOF"} +{"time":"2025-06-18T23:47:24.293322409+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/easy_r1/gokgllcc/file_stream\": read tcp 10.1.4.110:34406->172.67.193.61:443: read: connection timed out"} +{"time":"2025-06-18T23:55:12.153323353+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/easy_r1/gokgllcc/file_stream\": unexpected EOF"} +{"time":"2025-06-18T23:58:20.790254306+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/easy_r1/gokgllcc/file_stream\": read tcp 10.1.4.110:42558->172.67.193.61:443: read: connection reset by peer"} +{"time":"2025-06-18T23:59:35.686166744+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/easy_r1/gokgllcc/file_stream\": read tcp 10.1.4.110:59530->172.67.193.61:443: read: connection reset by peer"} +{"time":"2025-06-19T00:05:42.022104774+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/easy_r1/gokgllcc/file_stream\": read tcp 10.1.4.110:56430->172.67.193.61:443: read: connection timed out"} +{"time":"2025-06-19T00:08:46.85507911+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/easy_r1/gokgllcc/file_stream\": read tcp 10.1.4.110:59494->172.67.193.61:443: read: connection timed out"} +{"time":"2025-06-19T01:29:20.250295823+08:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-06-19T01:29:20.267000415+08:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-06-19T01:29:21.250479775+08:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"uploading output.log","runtime_seconds":0.457905197,"progress":"150.7KB/150.7KB"}],"total_operations":1}} +{"time":"2025-06-19T01:29:21.815288196+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-06-19T01:29:22.894812739+08:00","level":"INFO","msg":"stream: closing","id":"gokgllcc"} +{"time":"2025-06-19T01:29:22.894836426+08:00","level":"INFO","msg":"handler: closed","stream_id":"gokgllcc"} +{"time":"2025-06-19T01:29:22.89484691+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"gokgllcc"} +{"time":"2025-06-19T01:29:22.894853652+08:00","level":"INFO","msg":"sender: closed","stream_id":"gokgllcc"} +{"time":"2025-06-19T01:29:22.897970129+08:00","level":"INFO","msg":"stream: closed","id":"gokgllcc"} diff --git a/EasyR1/examples/wandb/debug.log b/EasyR1/examples/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..da52e54a527a6f52457c0ff558551ea69ddba574 --- /dev/null +++ b/EasyR1/examples/wandb/debug.log @@ -0,0 +1,28 @@ +2025-06-18 20:58:00,417 INFO MainThread:7254 [wandb_setup.py:_flush():81] Current SDK version is 0.20.1 +2025-06-18 20:58:00,417 INFO MainThread:7254 [wandb_setup.py:_flush():81] Configure stats pid to 7254 +2025-06-18 20:58:00,417 INFO MainThread:7254 [wandb_setup.py:_flush():81] Loading settings from /root/.config/wandb/settings +2025-06-18 20:58:00,417 INFO MainThread:7254 [wandb_setup.py:_flush():81] Loading settings from /nas/shared/kilab/wangyujia/EasyR1/examples/wandb/settings +2025-06-18 20:58:00,417 INFO MainThread:7254 [wandb_setup.py:_flush():81] Loading settings from environment variables +2025-06-18 20:58:00,417 INFO MainThread:7254 [wandb_init.py:setup_run_log_directory():703] Logging user logs to /nas/shared/kilab/wangyujia/EasyR1/examples/wandb/run-20250618_205800-gokgllcc/logs/debug.log +2025-06-18 20:58:00,418 INFO MainThread:7254 [wandb_init.py:setup_run_log_directory():704] Logging internal logs to /nas/shared/kilab/wangyujia/EasyR1/examples/wandb/run-20250618_205800-gokgllcc/logs/debug-internal.log +2025-06-18 20:58:00,418 INFO MainThread:7254 [wandb_init.py:init():831] calling init triggers +2025-06-18 20:58:00,418 INFO MainThread:7254 [wandb_init.py:init():836] wandb.init called with sweep_config: {} +config: {'data': {'train_files': '/nas/shared/kilab/wangyujia/rl_data/deeplocmulti@train', 'val_files': '/nas/shared/kilab/wangyujia/rl_data/deeplocmulti@validation', 'prompt_key': 'question', 'answer_key': 'answer', 'image_key': 'images', 'image_dir': None, 'max_prompt_length': 4096, 'max_response_length': 16384, 'rollout_batch_size': 128, 'val_batch_size': 256, 'format_prompt': '/nas/shared/kilab/wangyujia/EasyR1/examples/format_prompt/bio_format.jinja', 'override_chat_template': None, 'shuffle': True, 'seed': 1, 'min_pixels': 262144, 'max_pixels': 4194304, 'filter_overlong_prompts': True}, 'worker': {'hybrid_engine': True, 'actor': {'strategy': 'fsdp', 'global_batch_size': 64, 'micro_batch_size_per_device_for_update': 2, 'micro_batch_size_per_device_for_experience': 16, 'max_grad_norm': 1.0, 'clip_ratio_low': 0.2, 'clip_ratio_high': 0.3, 'clip_ratio_dual': 3.0, 'ppo_epochs': 1, 'padding_free': True, 'ulysses_sequence_parallel_size': 1, 'use_torch_compile': True, 'model': {'model_path': '/oss/wangyujia/BIO/pretrain_output/qwen2.5-7b-instruct-bio/bio_all/save1epoch/checkpoint-1300', 'tokenizer_path': '/oss/wangyujia/BIO/pretrain_output/qwen2.5-7b-instruct-bio/bio_all/save1epoch/checkpoint-1300', 'override_config': {}, 'enable_gradient_checkpointing': True, 'trust_remote_code': False, 'freeze_vision_tower': False}, 'optim': {'lr': 1e-06, 'betas': [0.9, 0.999], 'weight_decay': 0.01, 'strategy': 'adamw', 'lr_warmup_ratio': 0.0, 'min_lr_ratio': None, 'warmup_style': 'constant', 'training_steps': 31}, 'fsdp': {'enable_full_shard': True, 'enable_cpu_offload': False, 'enable_rank0_init': True, 'use_orig_params': False, 'torch_dtype': None, 'fsdp_size': -1, 'mp_param_dtype': 'bf16', 'mp_reduce_dtype': 'fp32', 'mp_buffer_dtype': 'fp32'}, 'offload': {'offload_params': True, 'offload_optimizer': True}, 'global_batch_size_per_device': -1, 'disable_kl': False, 'use_kl_loss': True, 'kl_penalty': 'low_var_kl', 'kl_coef': 0.01}, 'critic': {'strategy': 'fsdp', 'global_batch_size': 256, 'micro_batch_size_per_device_for_update': 4, 'micro_batch_size_per_device_for_experience': 16, 'max_grad_norm': 1.0, 'cliprange_value': 0.5, 'ppo_epochs': 1, 'padding_free': False, 'ulysses_sequence_parallel_size': 1, 'model': {'model_path': None, 'tokenizer_path': None, 'override_config': {}, 'enable_gradient_checkpointing': True, 'trust_remote_code': True, 'freeze_vision_tower': False}, 'optim': {'lr': 1e-06, 'betas': [0.9, 0.999], 'weight_decay': 0.01, 'strategy': 'adamw', 'lr_warmup_ratio': 0.0, 'min_lr_ratio': None, 'warmup_style': 'constant', 'training_steps': 31}, 'fsdp': {'enable_full_shard': True, 'enable_cpu_offload': False, 'enable_rank0_init': True, 'use_orig_params': False, 'torch_dtype': None, 'fsdp_size': -1, 'mp_param_dtype': 'bf16', 'mp_reduce_dtype': 'fp32', 'mp_buffer_dtype': 'fp32'}, 'offload': {'offload_params': False, 'offload_optimizer': False}, 'global_batch_size_per_device': -1}, 'ref': {'strategy': 'fsdp', 'fsdp': {'enable_full_shard': True, 'enable_cpu_offload': True, 'enable_rank0_init': True, 'use_orig_params': False, 'torch_dtype': None, 'fsdp_size': -1, 'mp_param_dtype': 'bf16', 'mp_reduce_dtype': 'fp32', 'mp_buffer_dtype': 'fp32'}, 'offload': {'offload_params': False, 'offload_optimizer': False}, 'micro_batch_size_per_device_for_experience': 16, 'padding_free': True, 'ulysses_sequence_parallel_size': 1, 'use_torch_compile': True}, 'reward': {'reward_type': 'batch', 'reward_function': '/nas/shared/kilab/wangyujia/EasyR1/examples/reward_function/bio.py', 'reward_function_kwargs': {}, 'skip_special_tokens': True, 'num_cpus': 1, 'reward_function_name': 'compute_score'}, 'rollout': {'name': 'vllm', 'n': 5, 'temperature': 1.0, 'top_p': 0.99, 'top_k': -1, 'seed': 1, 'limit_images': 0, 'dtype': 'bf16', 'gpu_memory_utilization': 0.6, 'ignore_eos': False, 'enforce_eager': False, 'enable_chunked_prefill': False, 'tensor_parallel_size': 1, 'max_model_len': None, 'max_num_batched_tokens': 24576, 'disable_log_stats': True, 'val_override_config': {'temperature': 0.5, 'n': 1}, 'prompt_length': 4096, 'response_length': 16384, 'trust_remote_code': False}}, 'algorithm': {'gamma': 1.0, 'lam': 1.0, 'adv_estimator': 'grpo', 'disable_kl': False, 'use_kl_loss': True, 'kl_penalty': 'low_var_kl', 'kl_coef': 0.01, 'kl_type': 'fixed', 'kl_horizon': 0.0, 'kl_target': 0.0}, 'trainer': {'total_epochs': 1, 'max_steps': None, 'project_name': 'easy_r1', 'experiment_name': 'qwen2.5_7b_bio_06182042', 'logger': ['console', 'wandb'], 'nnodes': 1, 'n_gpus_per_node': 8, 'critic_warmup': 0, 'val_freq': 5, 'val_before_train': True, 'val_only': False, 'val_generations_to_log': 3, 'save_freq': 5, 'save_limit': 3, 'save_checkpoint_path': '/oss/wangyujia/BIO/rl/qwen2.5_7b_bio_06182042', 'load_checkpoint_path': None}, '_wandb': {}} +2025-06-18 20:58:00,418 INFO MainThread:7254 [wandb_init.py:init():872] starting backend +2025-06-18 20:58:00,630 INFO MainThread:7254 [wandb_init.py:init():875] sending inform_init request +2025-06-18 20:58:00,633 INFO MainThread:7254 [wandb_init.py:init():883] backend started and connected +2025-06-18 20:58:00,637 INFO MainThread:7254 [wandb_init.py:init():956] updated telemetry +2025-06-18 20:58:00,638 INFO MainThread:7254 [wandb_init.py:init():980] communicating run to backend with 90.0 second timeout +2025-06-18 20:58:02,224 INFO MainThread:7254 [wandb_init.py:init():1032] starting run threads in backend +2025-06-18 20:58:02,474 INFO MainThread:7254 [wandb_run.py:_console_start():2453] atexit reg +2025-06-18 20:58:02,474 INFO MainThread:7254 [wandb_run.py:_redirect():2301] redirect: wrap_raw +2025-06-18 20:58:02,474 INFO MainThread:7254 [wandb_run.py:_redirect():2370] Wrapping output streams. +2025-06-18 20:58:02,474 INFO MainThread:7254 [wandb_run.py:_redirect():2393] Redirects installed. +2025-06-18 20:58:02,481 INFO MainThread:7254 [wandb_init.py:init():1078] run started, returning control to user process +2025-06-19 01:29:20,221 INFO MainThread:7254 [wandb_run.py:_finish():2219] finishing run gia0603yucca/easy_r1/gokgllcc +2025-06-19 01:29:20,240 INFO MainThread:7254 [wandb_run.py:_atexit_cleanup():2418] got exitcode: 0 +2025-06-19 01:29:20,246 INFO MainThread:7254 [wandb_run.py:_restore():2400] restore +2025-06-19 01:29:20,247 INFO MainThread:7254 [wandb_run.py:_restore():2406] restore done +2025-06-19 01:29:22,887 INFO MainThread:7254 [wandb_run.py:_footer_history_summary_info():4000] rendering history +2025-06-19 01:29:22,889 INFO MainThread:7254 [wandb_run.py:_footer_history_summary_info():4032] rendering summary +2025-06-19 01:29:22,890 INFO MainThread:7254 [wandb_run.py:_footer_sync_info():3961] logging synced files diff --git a/EasyR1/pyproject.toml b/EasyR1/pyproject.toml new file mode 100644 index 0000000000000000000000000000000000000000..2f579c19b4ec15bfb975ea1ad1d266d00a516dfe --- /dev/null +++ b/EasyR1/pyproject.toml @@ -0,0 +1,39 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "verl" +dynamic = [ + "version", + "dependencies", + "optional-dependencies", + "requires-python", + "authors", + "description", + "readme", + "license" +] + +[tool.ruff] +target-version = "py39" +line-length = 119 +indent-width = 4 + +[tool.ruff.lint] +ignore = ["C901", "E501", "E741", "W605", "C408"] +select = ["C", "E", "F", "I", "W", "RUF022"] + +[tool.ruff.lint.per-file-ignores] +"__init__.py" = ["E402", "F401", "F403", "F811"] + +[tool.ruff.lint.isort] +lines-after-imports = 2 +known-first-party = ["verl"] +known-third-party = ["torch", "transformers", "wandb"] + +[tool.ruff.format] +quote-style = "double" +indent-style = "space" +skip-magic-trailing-comma = false +line-ending = "auto" diff --git a/EasyR1/requirements.txt b/EasyR1/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..8543c231b9ec75fe9300417c1498d9328dfadbf2 --- /dev/null +++ b/EasyR1/requirements.txt @@ -0,0 +1,19 @@ +accelerate +codetiming +datasets +liger-kernel +mathruler +numpy +omegaconf +pandas +peft +pillow +pyarrow>=15.0.0 +pylatexenc +qwen-vl-utils +ray[default] +tensordict +torchdata +transformers>=4.51.0 +vllm>=0.7.3 +wandb diff --git a/EasyR1/setup.py b/EasyR1/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..b836e58c98f8ec15ddcfcf9f4f96daa71de0bf59 --- /dev/null +++ b/EasyR1/setup.py @@ -0,0 +1,61 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re + +from setuptools import find_packages, setup + + +def get_version() -> str: + with open(os.path.join("verl", "__init__.py"), encoding="utf-8") as f: + file_content = f.read() + pattern = r"__version__\W*=\W*\"([^\"]+)\"" + (version,) = re.findall(pattern, file_content) + return version + + +def get_requires() -> list[str]: + with open("requirements.txt", encoding="utf-8") as f: + file_content = f.read() + lines = [line.strip() for line in file_content.strip().split("\n") if not line.startswith("#")] + return lines + + +extra_require = { + "dev": ["pre-commit", "ruff"], +} + + +def main(): + setup( + name="verl", + version=get_version(), + description="An Efficient, Scalable, Multi-Modality RL Training Framework based on veRL", + long_description=open("README.md", encoding="utf-8").read(), + long_description_content_type="text/markdown", + author="verl", + author_email="zhangchi.usc1992@bytedance.com, gmsheng@connect.hku.hk, hiyouga@buaa.edu.cn", + license="Apache 2.0 License", + url="https://github.com/volcengine/verl", + package_dir={"": "."}, + packages=find_packages(where="."), + python_requires=">=3.9.0", + install_requires=get_requires(), + extras_require=extra_require, + ) + + +if __name__ == "__main__": + main()