Add files using upload-large-folder tool

BIO/ablation/metal_ion_binding_test.jsonl

@@ -0,0 +1,718 @@
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "0", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}
+{"reference_answer": "1", "generated_answer": "1"}

ProtT3/all_checkpoints/stage1_06221723/wandb/run-20250622_192645-zmuhvn72/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":42}}

ProtT3/all_checkpoints/stage1_06221723/wandb/run-20250622_192645-zmuhvn72/logs/debug-internal.log

@@ -0,0 +1,15 @@
+{"time":"2025-06-22T19:26:45.459589355+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.11","symlink path":"all_checkpoints/stage1_06221723/wandb/run-20250622_192645-zmuhvn72/logs/debug-core.log"}
+{"time":"2025-06-22T19:26:46.458621861+08:00","level":"INFO","msg":"created new stream","id":"zmuhvn72"}
+{"time":"2025-06-22T19:26:46.458658451+08:00","level":"INFO","msg":"stream: started","id":"zmuhvn72"}
+{"time":"2025-06-22T19:26:46.458694604+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"zmuhvn72"}
+{"time":"2025-06-22T19:26:46.458731318+08:00","level":"INFO","msg":"sender: started","stream_id":"zmuhvn72"}
+{"time":"2025-06-22T19:26:46.45883181+08:00","level":"INFO","msg":"handler: started","stream_id":"zmuhvn72"}
+{"time":"2025-06-22T19:26:47.693527925+08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2025-06-22T19:27:27.828788085+08:00","level":"INFO","msg":"stream: closing","id":"zmuhvn72"}
+{"time":"2025-06-22T19:27:27.828880453+08:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2025-06-22T19:27:27.829610297+08:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2025-06-22T19:27:29.428896311+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-06-22T19:27:30.389945053+08:00","level":"INFO","msg":"handler: closed","stream_id":"zmuhvn72"}
+{"time":"2025-06-22T19:27:30.38999824+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"zmuhvn72"}
+{"time":"2025-06-22T19:27:30.390015358+08:00","level":"INFO","msg":"sender: closed","stream_id":"zmuhvn72"}
+{"time":"2025-06-22T19:27:30.395367992+08:00","level":"INFO","msg":"stream: closed","id":"zmuhvn72"}

ProtT3/all_checkpoints/stage1_06221723/wandb/run-20250622_192645-zmuhvn72/logs/debug.log

@@ -0,0 +1,24 @@
+2025-06-22 19:26:45,451 INFO    MainThread:104393 [wandb_setup.py:_flush():70] Current SDK version is 0.19.11
+2025-06-22 19:26:45,451 INFO    MainThread:104393 [wandb_setup.py:_flush():70] Configure stats pid to 104393
+2025-06-22 19:26:45,451 INFO    MainThread:104393 [wandb_setup.py:_flush():70] Loading settings from /root/.config/wandb/settings
+2025-06-22 19:26:45,451 INFO    MainThread:104393 [wandb_setup.py:_flush():70] Loading settings from /nas/shared/kilab/wangyujia/ProtT3/wandb/settings
+2025-06-22 19:26:45,451 INFO    MainThread:104393 [wandb_setup.py:_flush():70] Loading settings from environment variables
+2025-06-22 19:26:45,451 INFO    MainThread:104393 [wandb_init.py:setup_run_log_directory():724] Logging user logs to ./all_checkpoints/stage1_06221723/wandb/run-20250622_192645-zmuhvn72/logs/debug.log
+2025-06-22 19:26:45,451 INFO    MainThread:104393 [wandb_init.py:setup_run_log_directory():725] Logging internal logs to ./all_checkpoints/stage1_06221723/wandb/run-20250622_192645-zmuhvn72/logs/debug-internal.log
+2025-06-22 19:26:45,451 INFO    MainThread:104393 [wandb_init.py:init():852] calling init triggers
+2025-06-22 19:26:45,451 INFO    MainThread:104393 [wandb_init.py:init():857] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-06-22 19:26:45,451 INFO    MainThread:104393 [wandb_init.py:init():893] starting backend
+2025-06-22 19:26:45,451 INFO    MainThread:104393 [wandb_init.py:init():897] sending inform_init request
+2025-06-22 19:26:45,453 INFO    MainThread:104393 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-06-22 19:26:45,454 INFO    MainThread:104393 [wandb_init.py:init():907] backend started and connected
+2025-06-22 19:26:45,458 INFO    MainThread:104393 [wandb_init.py:init():1005] updated telemetry
+2025-06-22 19:26:45,461 INFO    MainThread:104393 [wandb_init.py:init():1029] communicating run to backend with 90.0 second timeout
+2025-06-22 19:26:47,645 INFO    MainThread:104393 [wandb_init.py:init():1104] starting run threads in backend
+2025-06-22 19:26:47,826 INFO    MainThread:104393 [wandb_run.py:_console_start():2573] atexit reg
+2025-06-22 19:26:47,826 INFO    MainThread:104393 [wandb_run.py:_redirect():2421] redirect: wrap_raw
+2025-06-22 19:26:47,829 INFO    MainThread:104393 [wandb_run.py:_redirect():2490] Wrapping output streams.
+2025-06-22 19:26:47,830 INFO    MainThread:104393 [wandb_run.py:_redirect():2513] Redirects installed.
+2025-06-22 19:26:47,831 INFO    MainThread:104393 [wandb_init.py:init():1150] run started, returning control to user process
+2025-06-22 19:26:54,211 INFO    MainThread:104393 [wandb_run.py:_config_callback():1436] config_cb None None {'filename': 'stage1_06221723', 'seed': 42, 'mode': 'train', 'strategy': 'deepspeed', 'accelerator': 'gpu', 'devices': '0,1,2,3,4,5,6,7', 'precision': 'bf16-mixed', 'max_epochs': 20, 'check_val_every_n_epoch': 1, 'use_wandb_logger': True, 'mix_dataset': True, 'temperature': 0.1, 'save_every_n_epochs': 5, 'ptm': True, 'lm': True, 'rerank_cand_num': 128, 'plm_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m', 'plm_tune': 'freeze', 'load_4bit': False, 'pool_size': 0, 'bert_hidden_dim': 768, 'bert_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft', 'projection_dim': 256, 'cross_attention_freq': 2, 'num_query_token': 8, 'weight_decay': 0.05, 'init_lr': 0.0001, 'min_lr': 1e-05, 'warmup_lr': 1e-06, 'warmup_steps': 1000, 'lr_decay_rate': 0.9, 'scheduler': 'linear_warmup_cosine_lr', 'init_checkpoint': '', 'retrieval_eval_epoch': 10, 'num_workers': 8, 'batch_size': 32, 'match_batch_size': 64, 'root': 'data', 'text_max_len': 128, 'prot_max_len': 1024, 'prot_aug': 'None'}
+2025-06-22 19:27:27,827 INFO    MsgRouterThr:104393 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles.

ProtT3/all_checkpoints/stage1_06261435/wandb/debug-internal.log

@@ -0,0 +1,15 @@
+{"time":"2025-06-26T14:47:31.30118788+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.11","symlink path":"all_checkpoints/stage1_06261435/wandb/run-20250626_144731-1bz1vyyf/logs/debug-core.log"}
+{"time":"2025-06-26T14:47:33.763139479+08:00","level":"INFO","msg":"created new stream","id":"1bz1vyyf"}
+{"time":"2025-06-26T14:47:33.763180996+08:00","level":"INFO","msg":"stream: started","id":"1bz1vyyf"}
+{"time":"2025-06-26T14:47:33.76320552+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"1bz1vyyf"}
+{"time":"2025-06-26T14:47:33.763263895+08:00","level":"INFO","msg":"handler: started","stream_id":"1bz1vyyf"}
+{"time":"2025-06-26T14:47:33.763302435+08:00","level":"INFO","msg":"sender: started","stream_id":"1bz1vyyf"}
+{"time":"2025-06-26T14:47:35.049823143+08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2025-06-26T14:59:24.18296941+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06261435/1bz1vyyf/file_stream\": EOF"}
+{"time":"2025-06-26T15:47:20.217171547+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-26T15:47:52.39492821+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-26T15:48:27.272812301+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-06-26T15:49:05.914803533+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-26T15:49:51.452193247+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06261435/1bz1vyyf/file_stream\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-26T15:49:52.900302871+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-06-26T15:50:55.486728897+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}

ProtT3/all_checkpoints/stage1_06261435/wandb/debug.log

@@ -0,0 +1,23 @@
+2025-06-26 14:47:31,292 INFO    MainThread:13641 [wandb_setup.py:_flush():70] Current SDK version is 0.19.11
+2025-06-26 14:47:31,292 INFO    MainThread:13641 [wandb_setup.py:_flush():70] Configure stats pid to 13641
+2025-06-26 14:47:31,292 INFO    MainThread:13641 [wandb_setup.py:_flush():70] Loading settings from /root/.config/wandb/settings
+2025-06-26 14:47:31,292 INFO    MainThread:13641 [wandb_setup.py:_flush():70] Loading settings from /nas/shared/kilab/wangyujia/ProtT3/wandb/settings
+2025-06-26 14:47:31,292 INFO    MainThread:13641 [wandb_setup.py:_flush():70] Loading settings from environment variables
+2025-06-26 14:47:31,292 INFO    MainThread:13641 [wandb_init.py:setup_run_log_directory():724] Logging user logs to ./all_checkpoints/stage1_06261435/wandb/run-20250626_144731-1bz1vyyf/logs/debug.log
+2025-06-26 14:47:31,292 INFO    MainThread:13641 [wandb_init.py:setup_run_log_directory():725] Logging internal logs to ./all_checkpoints/stage1_06261435/wandb/run-20250626_144731-1bz1vyyf/logs/debug-internal.log
+2025-06-26 14:47:31,292 INFO    MainThread:13641 [wandb_init.py:init():852] calling init triggers
+2025-06-26 14:47:31,292 INFO    MainThread:13641 [wandb_init.py:init():857] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-06-26 14:47:31,292 INFO    MainThread:13641 [wandb_init.py:init():893] starting backend
+2025-06-26 14:47:31,292 INFO    MainThread:13641 [wandb_init.py:init():897] sending inform_init request
+2025-06-26 14:47:31,294 INFO    MainThread:13641 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-06-26 14:47:31,296 INFO    MainThread:13641 [wandb_init.py:init():907] backend started and connected
+2025-06-26 14:47:31,300 INFO    MainThread:13641 [wandb_init.py:init():1005] updated telemetry
+2025-06-26 14:47:31,303 INFO    MainThread:13641 [wandb_init.py:init():1029] communicating run to backend with 90.0 second timeout
+2025-06-26 14:47:35,041 INFO    MainThread:13641 [wandb_init.py:init():1104] starting run threads in backend
+2025-06-26 14:47:35,187 INFO    MainThread:13641 [wandb_run.py:_console_start():2573] atexit reg
+2025-06-26 14:47:35,187 INFO    MainThread:13641 [wandb_run.py:_redirect():2421] redirect: wrap_raw
+2025-06-26 14:47:35,192 INFO    MainThread:13641 [wandb_run.py:_redirect():2490] Wrapping output streams.
+2025-06-26 14:47:35,192 INFO    MainThread:13641 [wandb_run.py:_redirect():2513] Redirects installed.
+2025-06-26 14:47:35,197 INFO    MainThread:13641 [wandb_init.py:init():1150] run started, returning control to user process
+2025-06-26 14:47:41,914 INFO    MainThread:13641 [wandb_run.py:_config_callback():1436] config_cb None None {'filename': 'stage1_06261435', 'seed': 42, 'mode': 'train', 'strategy': 'deepspeed', 'accelerator': 'gpu', 'devices': '0,1,2,3,4,5,6,7', 'precision': 'bf16-mixed', 'max_epochs': 30, 'check_val_every_n_epoch': 1, 'use_wandb_logger': True, 'mix_dataset': True, 'temperature': 0.1, 'save_every_n_epochs': 5, 'ptm': True, 'lm': True, 'rerank_cand_num': 128, 'plm_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m', 'plm_tune': 'freeze', 'load_4bit': False, 'pool_size': 0, 'bert_hidden_dim': 768, 'bert_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft', 'projection_dim': 256, 'cross_attention_freq': 2, 'num_query_token': 8, 'weight_decay': 0.05, 'init_lr': 0.0001, 'min_lr': 1e-05, 'warmup_lr': 1e-06, 'warmup_steps': 1000, 'lr_decay_rate': 0.9, 'scheduler': 'linear_warmup_cosine_lr', 'init_checkpoint': '', 'retrieval_eval_epoch': 10, 'num_workers': 8, 'batch_size': 160, 'match_batch_size': 64, 'root': 'data', 'text_max_len': 128, 'prot_max_len': 1024, 'prot_aug': 'None'}

ProtT3/all_checkpoints/stage1_06261435/wandb/run-20250626_143705-0goy7nyw/logs/debug-internal.log

@@ -0,0 +1,3 @@
+{"time":"2025-06-26T14:37:05.188561402+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.11","symlink path":"all_checkpoints/stage1_06261435/wandb/run-20250626_143705-0goy7nyw/logs/debug-core.log"}
+{"time":"2025-06-26T14:37:35.294367385+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-26T14:38:07.797500016+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}

ProtT3/all_checkpoints/stage1_06261435/wandb/run-20250626_143705-0goy7nyw/logs/debug.log

@@ -0,0 +1,94 @@
+2025-06-26 14:37:05,174 INFO    MainThread:2555 [wandb_setup.py:_flush():70] Current SDK version is 0.19.11
+2025-06-26 14:37:05,174 INFO    MainThread:2555 [wandb_setup.py:_flush():70] Configure stats pid to 2555
+2025-06-26 14:37:05,174 INFO    MainThread:2555 [wandb_setup.py:_flush():70] Loading settings from /root/.config/wandb/settings
+2025-06-26 14:37:05,174 INFO    MainThread:2555 [wandb_setup.py:_flush():70] Loading settings from /nas/shared/kilab/wangyujia/ProtT3/wandb/settings
+2025-06-26 14:37:05,174 INFO    MainThread:2555 [wandb_setup.py:_flush():70] Loading settings from environment variables
+2025-06-26 14:37:05,174 INFO    MainThread:2555 [wandb_init.py:setup_run_log_directory():724] Logging user logs to ./all_checkpoints/stage1_06261435/wandb/run-20250626_143705-0goy7nyw/logs/debug.log
+2025-06-26 14:37:05,174 INFO    MainThread:2555 [wandb_init.py:setup_run_log_directory():725] Logging internal logs to ./all_checkpoints/stage1_06261435/wandb/run-20250626_143705-0goy7nyw/logs/debug-internal.log
+2025-06-26 14:37:05,174 INFO    MainThread:2555 [wandb_init.py:init():852] calling init triggers
+2025-06-26 14:37:05,174 INFO    MainThread:2555 [wandb_init.py:init():857] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-06-26 14:37:05,174 INFO    MainThread:2555 [wandb_init.py:init():893] starting backend
+2025-06-26 14:37:05,174 INFO    MainThread:2555 [wandb_init.py:init():897] sending inform_init request
+2025-06-26 14:37:05,178 INFO    MainThread:2555 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-06-26 14:37:05,178 INFO    MainThread:2555 [wandb_init.py:init():907] backend started and connected
+2025-06-26 14:37:05,180 INFO    MainThread:2555 [wandb_init.py:init():1005] updated telemetry
+2025-06-26 14:37:05,182 INFO    MainThread:2555 [wandb_init.py:init():1029] communicating run to backend with 90.0 second timeout
+2025-06-26 14:37:37,418 INFO    Thread-3 (wrapped_target):2555 [retry.py:__call__():175] [no run ID] Retry attempt failed:
+Traceback (most recent call last):
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/urllib3/connection.py", line 198, in _new_conn
+    sock = connection.create_connection(
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/urllib3/util/connection.py", line 85, in create_connection
+    raise err
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/urllib3/util/connection.py", line 73, in create_connection
+    sock.connect(sa)
+TimeoutError: timed out
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/urllib3/connectionpool.py", line 787, in urlopen
+    response = self._make_request(
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/urllib3/connectionpool.py", line 488, in _make_request
+    raise new_e
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/urllib3/connectionpool.py", line 464, in _make_request
+    self._validate_conn(conn)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/urllib3/connectionpool.py", line 1093, in _validate_conn
+    conn.connect()
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/urllib3/connection.py", line 704, in connect
+    self.sock = sock = self._new_conn()
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/urllib3/connection.py", line 207, in _new_conn
+    raise ConnectTimeoutError(
+urllib3.exceptions.ConnectTimeoutError: (<urllib3.connection.HTTPSConnection object at 0x7fa5a7f7b9a0>, 'Connection to api.wandb.ai timed out. (connect timeout=20)')
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/requests/adapters.py", line 667, in send
+    resp = conn.urlopen(
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/urllib3/connectionpool.py", line 841, in urlopen
+    retries = retries.increment(
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/urllib3/util/retry.py", line 519, in increment
+    raise MaxRetryError(_pool, url, reason) from reason  # type: ignore[arg-type]
+urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='api.wandb.ai', port=443): Max retries exceeded with url: /graphql (Caused by ConnectTimeoutError(<urllib3.connection.HTTPSConnection object at 0x7fa5a7f7b9a0>, 'Connection to api.wandb.ai timed out. (connect timeout=20)'))
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/wandb/sdk/lib/retry.py", line 134, in __call__
+    result = self._call_fn(*args, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/wandb/sdk/internal/internal_api.py", line 398, in execute
+    return self.client.execute(*args, **kwargs)  # type: ignore
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/wandb/vendor/gql-0.2.0/wandb_gql/client.py", line 52, in execute
+    result = self._get_result(document, *args, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/wandb/vendor/gql-0.2.0/wandb_gql/client.py", line 60, in _get_result
+    return self.transport.execute(document, *args, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/wandb/sdk/lib/gql_request.py", line 58, in execute
+    request = self.session.post(self.url, **post_args)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/requests/sessions.py", line 637, in post
+    return self.request("POST", url, data=data, json=json, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/requests/sessions.py", line 589, in request
+    resp = self.send(prep, **send_kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/requests/sessions.py", line 703, in send
+    r = adapter.send(request, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/requests/adapters.py", line 688, in send
+    raise ConnectTimeout(e, request=request)
+requests.exceptions.ConnectTimeout: HTTPSConnectionPool(host='api.wandb.ai', port=443): Max retries exceeded with url: /graphql (Caused by ConnectTimeoutError(<urllib3.connection.HTTPSConnection object at 0x7fa5a7f7b9a0>, 'Connection to api.wandb.ai timed out. (connect timeout=20)'))
+2025-06-26 14:38:28,356 WARNING MainThread:2555 [wandb_init.py:init():1681] [no run ID] interrupted
+Traceback (most recent call last):
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/wandb/sdk/wandb_init.py", line 1677, in init
+    return wi.init(run_settings, run_config, run_printer)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/wandb/sdk/wandb_init.py", line 1055, in init
+    result = wait_with_progress(
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/wandb/sdk/mailbox/wait_with_progress.py", line 24, in wait_with_progress
+    return wait_all_with_progress(
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/wandb/sdk/mailbox/wait_with_progress.py", line 87, in wait_all_with_progress
+    return asyncio_compat.run(progress_loop_with_timeout)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/wandb/sdk/lib/asyncio_compat.py", line 30, in run
+    return future.result()
+  File "/root/miniconda3/envs/protT3/lib/python3.10/concurrent/futures/_base.py", line 440, in result
+    self._condition.wait(timeout)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/threading.py", line 320, in wait
+    waiter.acquire()
+KeyboardInterrupt
+2025-06-26 14:38:29,216 INFO    MsgRouterThr:2555 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 2 handles.

ProtT3/all_checkpoints/stage1_06261435/wandb/run-20250626_144000-coknhy79/files/output.log

@@ -0,0 +1,19 @@
+/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:654: Checkpoint directory /nas/shared/kilab/wangyujia/ProtT3/all_checkpoints/stage1_06261435 exists and is not empty.
+Enabling DeepSpeed BF16. Model parameters and inputs will be cast to `bfloat16`.
+LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]
+
+  | Name         | Type         | Params | Mode
+------------------------------------------------------
+0 | blip2qformer | Blip2Qformer | 327 M  | train
+------------------------------------------------------
+179 M     Trainable params
+147 M     Non-trainable params
+327 M     Total params
+1,309.467 Total estimated model params size (MB)
+5         Modules in train mode
+926       Modules in eval mode
+Epoch 0:   3%|██████▉                                                                                                                                                                                                                                        | 48/1665 [00:58<32:34,  0.83it/s, v_num=hy79]
+/nas/shared/kilab/wangyujia/ProtT3/model/blip2qformer.py:220: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(enable_autocast, dtype=torch.float32):
+
+Detected KeyboardInterrupt, attempting graceful shutdown ...

ProtT3/all_checkpoints/stage1_06261435/wandb/run-20250626_144000-coknhy79/files/requirements.txt

@@ -0,0 +1,225 @@
+attrs==25.3.0
+tqdm==4.67.1
+langcodes==3.5.0
+nvidia-cublas-cu12==12.4.5.8
+tifffile==2025.5.10
+nvidia-cufile-cu12==1.11.1.6
+nltk==3.9.1
+salesforce-lavis==1.0.2
+tzdata==2025.2
+pyparsing==3.2.3
+six==1.17.0
+python-dateutil==2.9.0.post0
+pandas==2.2.3
+pytorch-lightning==2.5.1.post0
+blinker==1.9.0
+opencv-python-headless==4.5.5.64
+nvidia-cusparse-cu12==12.3.1.170
+pytz==2025.2
+async-timeout==5.0.1
+pillow==11.2.1
+parso==0.8.4
+joblib==1.5.1
+contourpy==1.3.2
+triton==3.2.0
+marisa-trie==1.2.1
+PyYAML==6.0.2
+regex==2024.11.6
+idna==3.10
+nvidia-curand-cu12==10.3.5.147
+rpds-py==0.25.1
+aiosignal==1.3.2
+srsly==2.5.1
+confection==0.1.5
+typing-inspection==0.4.1
+packaging==24.2
+distlib==0.3.9
+networkx==3.4.2
+absl-py==2.2.2
+yarl==1.20.0
+lightning-utilities==0.14.3
+executing==2.2.0
+pycocoevalcap==1.2
+wheel==0.45.1
+nvidia-ml-py==12.575.51
+cycler==0.12.1
+wrapt==1.17.2
+jsonschema-specifications==2025.4.1
+protobuf==6.31.0
+mpmath==1.3.0
+certifi==2025.4.26
+py-cpuinfo==9.0.0
+contexttimer==0.3.3
+watchdog==6.0.0
+pexpect==4.9.0
+webencodings==0.5.1
+hf-xet==1.1.2
+cymem==2.0.11
+requests==2.32.3
+timm==0.4.12
+omegaconf==2.3.0
+nvidia-nvjitlink-cu12==12.4.127
+webdataset==0.2.111
+nodeenv==1.9.1
+frozenlist==1.6.0
+annotated-types==0.7.0
+matplotlib-inline==0.1.7
+urllib3==2.4.0
+rich==14.0.0
+GitPython==3.1.44
+lazy_loader==0.4
+msgpack==1.1.0
+prompt_toolkit==3.0.51
+fonttools==4.58.0
+multidict==6.4.4
+blis==1.3.0
+thinc==8.3.6
+nvidia-nvtx-cu12==12.4.127
+torchmetrics==1.7.1
+weasel==0.4.1
+numpy==2.2.6
+cachetools==5.5.2
+Jinja2==3.1.6
+matplotlib==3.10.3
+nvidia-cudnn-cu12==9.1.0.70
+Pygments==2.19.1
+tornado==6.5.1
+scipy==1.15.3
+rouge_score==0.1.2
+cloudpathlib==0.21.1
+jedi==0.19.2
+referencing==0.36.2
+decord==0.6.0
+setuptools==78.1.1
+mdurl==0.1.2
+identify==2.6.12
+python-slugify==8.0.4
+portalocker==3.1.1
+catalogue==2.0.10
+platformdirs==4.3.8
+antlr4-python3-runtime==4.9.3
+nvidia-cusolver-cu12==11.6.1.9
+kaggle==1.7.4.5
+pydeck==0.9.1
+pydantic==2.11.5
+nvidia-cufft-cu12==11.2.1.3
+pyarrow==20.0.0
+nvidia-nccl-cu12==2.21.5
+markdown-it-py==3.0.0
+gitdb==4.0.12
+altair==5.5.0
+torchvision==0.21.0
+python-magic==0.4.27
+iopath==0.1.10
+smart-open==7.1.0
+torch==2.6.0
+pycocotools==2.0.8
+fairscale==0.4.4
+traitlets==5.14.3
+pure_eval==0.2.3
+sympy==1.13.1
+nvidia-cusparselt-cu12==0.6.2
+imageio==2.37.0
+stack-data==0.6.3
+shellingham==1.5.4
+nvidia-cuda-runtime-cu12==12.4.127
+einops==0.8.1
+tenacity==9.1.2
+virtualenv==20.31.2
+ptyprocess==0.7.0
+cfgv==3.4.0
+pre_commit==4.2.0
+language_data==1.3.0
+typing_extensions==4.13.2
+propcache==0.3.1
+nvidia-cuda-cupti-cu12==12.4.127
+safetensors==0.5.3
+text-unidecode==1.3
+wcwidth==0.2.13
+charset-normalizer==3.4.2
+aiohappyeyeballs==2.6.1
+ipython==8.36.0
+streamlit==1.45.1
+asttokens==3.0.0
+psutil==7.0.0
+smmap==5.0.2
+exceptiongroup==1.3.0
+murmurhash==1.0.13
+filelock==3.18.0
+plotly==6.1.1
+hjson==3.1.0
+pydantic_core==2.33.2
+ninja==1.11.1.4
+kiwisolver==1.4.8
+spacy-legacy==3.0.12
+opendatasets==0.1.22
+decorator==5.2.1
+spacy==3.8.7
+wasabi==1.1.3
+sentencepiece==0.2.0
+toml==0.10.2
+scikit-image==0.25.2
+deepspeed==0.16.10+b666844f
+ftfy==6.3.1
+bleach==6.2.0
+nvidia-cuda-nvrtc-cu12==12.4.127
+spacy-loggers==1.0.5
+MarkupSafe==3.0.2
+braceexpand==0.1.7
+oss2==2.15.0
+preshed==3.0.10
+transformers==4.52.3
+aiohttp==3.12.2
+web.py==0.62
+threadpoolctl==3.6.0
+jaraco.functools==4.1.0
+wandb==0.19.11
+sentry-sdk==2.29.1
+tokenizers==0.21.1
+fsspec==2025.3.0
+flash-attn==2.7.1.post1
+opendelta==0.3.2
+opencv-python==4.11.0.86
+click==8.2.1
+docker-pycreds==0.4.0
+typer==0.16.0
+xxhash==3.5.0
+pathlib==1.0.1
+dill==0.3.8
+crcmod==1.7
+bigmodelvis==0.0.1
+datasets==3.6.0
+pycryptodome==3.23.0
+jsonschema==4.24.0
+aliyun-python-sdk-core==2.16.0
+jmespath==0.10.0
+more-itertools==10.7.0
+scikit-learn==1.6.1
+huggingface-hub==0.32.1
+cryptography==45.0.3
+pycparser==2.22
+yacs==0.1.8
+aliyun-python-sdk-kms==2.16.5
+cffi==1.17.1
+delta-center-client==0.0.4
+multiprocess==0.70.16
+setproctitle==1.3.6
+narwhals==1.41.0
+pip==25.1.1
+cheroot==10.0.1
+jaraco.context==5.3.0
+more-itertools==10.3.0
+jaraco.functools==4.0.1
+jaraco.text==3.12.1
+platformdirs==4.2.2
+packaging==24.2
+wheel==0.45.1
+zipp==3.19.2
+inflect==7.3.1
+autocommand==2.2.2
+typeguard==4.3.0
+jaraco.collections==5.1.0
+backports.tarfile==1.2.0
+tomli==2.0.1
+importlib_metadata==8.0.0
+typing_extensions==4.12.2

ProtT3/all_checkpoints/stage1_06261435/wandb/run-20250626_144000-coknhy79/files/wandb-metadata.json

@@ -0,0 +1,98 @@
+{
+  "os": "Linux-5.10.134-008.16.kangaroo.al8.x86_64-x86_64-with-glibc2.35",
+  "python": "CPython 3.10.0",
+  "startedAt": "2025-06-26T06:40:00.459849Z",
+  "args": [
+    "--devices",
+    "0,1,2,3,4,5,6,7",
+    "--mode",
+    "train",
+    "--filename",
+    "stage1_06261435",
+    "--num_query_token",
+    "8",
+    "--plm_name",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m",
+    "--bert_name",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft",
+    "--save_every_n_epochs",
+    "5",
+    "--max_epochs",
+    "30",
+    "--batch_size",
+    "64",
+    "--precision",
+    "bf16-mixed",
+    "--mix_dataset",
+    "--num_workers",
+    "8",
+    "--use_wandb_logger"
+  ],
+  "program": "/nas/shared/kilab/wangyujia/ProtT3/stage1.py",
+  "codePath": "stage1.py",
+  "email": "gia0603yucca@gmail.com",
+  "root": "./all_checkpoints/stage1_06261435/",
+  "host": "dsw-265304-cd576ddc5-gh74w",
+  "executable": "/root/miniconda3/envs/protT3/bin/python",
+  "codePathLocal": "stage1.py",
+  "cpu_count": 64,
+  "cpu_count_logical": 64,
+  "gpu": "NVIDIA A800-SXM4-80GB",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1623302262784",
+      "used": "1290833920"
+    }
+  },
+  "memory": {
+    "total": "549755813888"
+  },
+  "cpu": {
+    "count": 64,
+    "countLogical": 64
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    }
+  ],
+  "cudaVersion": "12.1"
+}

ProtT3/all_checkpoints/stage1_06261435/wandb/run-20250626_144000-coknhy79/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":87}}

ProtT3/all_checkpoints/stage1_06261435/wandb/run-20250626_144000-coknhy79/logs/debug-internal.log

@@ -0,0 +1,12 @@
+{"time":"2025-06-26T14:40:00.459807265+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.11","symlink path":"all_checkpoints/stage1_06261435/wandb/run-20250626_144000-coknhy79/logs/debug-core.log"}
+{"time":"2025-06-26T14:40:02.083580433+08:00","level":"INFO","msg":"created new stream","id":"coknhy79"}
+{"time":"2025-06-26T14:40:02.083623333+08:00","level":"INFO","msg":"stream: started","id":"coknhy79"}
+{"time":"2025-06-26T14:40:02.083652289+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"coknhy79"}
+{"time":"2025-06-26T14:40:02.083685885+08:00","level":"INFO","msg":"sender: started","stream_id":"coknhy79"}
+{"time":"2025-06-26T14:40:02.083706361+08:00","level":"INFO","msg":"handler: started","stream_id":"coknhy79"}
+{"time":"2025-06-26T14:40:05.653872904+08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2025-06-26T14:40:21.95203173+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/stage1_06261435/coknhy79/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250626%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250626T064005Z&X-Goog-Expires=86399&X-Goog-Signature=2ebb3b79b72a33f0dea008298dd64586dedd8e6bc4d1254ca4ee427dd9bc9031e9639976842b04c6f1e621bb4977fcba6d4c94c21e92008c662e898226494154965b113c395f94636cbca6cfd2249046ec06117750eebbc838bade9d9fa21ae04a11b6eb67dacc0319fb3bb297deca6c80a588a6a41a7b1f22b20c94a59d1f800926208ec84aa369a0a265f5cb9507e60248cf60d0ca950966ead524dec91e347bdb50ed95b2daa080e4e381b4e3dd1d85267b1297dc3c33d3bc683cac980ad150cf1033532af34d80aef288235efe863a3246dad1f2b656d6dbea37ff48485a03307ab8f7d65f6827ff79a95ebc8be7283ace09ee4bdd580361dd389452900a&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.8.160:47468->142.250.73.155:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/stage1_06261435/coknhy79/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250626%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250626T064005Z&X-Goog-Expires=86399&X-Goog-Signature=2ebb3b79b72a33f0dea008298dd64586dedd8e6bc4d1254ca4ee427dd9bc9031e9639976842b04c6f1e621bb4977fcba6d4c94c21e92008c662e898226494154965b113c395f94636cbca6cfd2249046ec06117750eebbc838bade9d9fa21ae04a11b6eb67dacc0319fb3bb297deca6c80a588a6a41a7b1f22b20c94a59d1f800926208ec84aa369a0a265f5cb9507e60248cf60d0ca950966ead524dec91e347bdb50ed95b2daa080e4e381b4e3dd1d85267b1297dc3c33d3bc683cac980ad150cf1033532af34d80aef288235efe863a3246dad1f2b656d6dbea37ff48485a03307ab8f7d65f6827ff79a95ebc8be7283ace09ee4bdd580361dd389452900a&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-06-26T14:40:21.952032953+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/stage1_06261435/coknhy79/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250626%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250626T064006Z&X-Goog-Expires=86399&X-Goog-Signature=78ae20776b7f86131b9272da30634723f339c53fbe35baa579e4d622bae104bf8b38c0c3cd5e64009eab167fa1132f4e5d02a2124a12f56f53a2fa10277fb6a7c582a81981d7372e6f2b2b33ff175bb4af1fc83ed04c00275ef4e90cec3133057f39515a0b0b4d55f5e5257bb5e28f9d142c819f9f4848bd11229d9c73dee989c0f69232f269e7efca958b28ec14404ae411af486b22b4b581a6f37ad7a8593025d9d20519451abe9ea912dac4130082c228f58687eb52f37b0911bf5fff35f7c2f75597dc0beebc355bb0a83b0f71cb06e8fadec0b9f798cf0da7e329a991260240180d651b91d4129bbd2f503247ad12cbb6e080076b2824d51383d713ed3b&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.8.160:47468->142.250.73.155:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/stage1_06261435/coknhy79/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250626%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250626T064006Z&X-Goog-Expires=86399&X-Goog-Signature=78ae20776b7f86131b9272da30634723f339c53fbe35baa579e4d622bae104bf8b38c0c3cd5e64009eab167fa1132f4e5d02a2124a12f56f53a2fa10277fb6a7c582a81981d7372e6f2b2b33ff175bb4af1fc83ed04c00275ef4e90cec3133057f39515a0b0b4d55f5e5257bb5e28f9d142c819f9f4848bd11229d9c73dee989c0f69232f269e7efca958b28ec14404ae411af486b22b4b581a6f37ad7a8593025d9d20519451abe9ea912dac4130082c228f58687eb52f37b0911bf5fff35f7c2f75597dc0beebc355bb0a83b0f71cb06e8fadec0b9f798cf0da7e329a991260240180d651b91d4129bbd2f503247ad12cbb6e080076b2824d51383d713ed3b&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-06-26T14:41:27.871959935+08:00","level":"INFO","msg":"stream: closing","id":"coknhy79"}
+{"time":"2025-06-26T14:41:27.872028784+08:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2025-06-26T14:41:27.940516252+08:00","level":"INFO","msg":"Stopped system monitor"}

ProtT3/all_checkpoints/stage1_06261435/wandb/run-20250626_144000-coknhy79/logs/debug.log

@@ -0,0 +1,24 @@
+2025-06-26 14:40:00,443 INFO    MainThread:5872 [wandb_setup.py:_flush():70] Current SDK version is 0.19.11
+2025-06-26 14:40:00,443 INFO    MainThread:5872 [wandb_setup.py:_flush():70] Configure stats pid to 5872
+2025-06-26 14:40:00,443 INFO    MainThread:5872 [wandb_setup.py:_flush():70] Loading settings from /root/.config/wandb/settings
+2025-06-26 14:40:00,443 INFO    MainThread:5872 [wandb_setup.py:_flush():70] Loading settings from /nas/shared/kilab/wangyujia/ProtT3/wandb/settings
+2025-06-26 14:40:00,443 INFO    MainThread:5872 [wandb_setup.py:_flush():70] Loading settings from environment variables
+2025-06-26 14:40:00,444 INFO    MainThread:5872 [wandb_init.py:setup_run_log_directory():724] Logging user logs to ./all_checkpoints/stage1_06261435/wandb/run-20250626_144000-coknhy79/logs/debug.log
+2025-06-26 14:40:00,444 INFO    MainThread:5872 [wandb_init.py:setup_run_log_directory():725] Logging internal logs to ./all_checkpoints/stage1_06261435/wandb/run-20250626_144000-coknhy79/logs/debug-internal.log
+2025-06-26 14:40:00,444 INFO    MainThread:5872 [wandb_init.py:init():852] calling init triggers
+2025-06-26 14:40:00,444 INFO    MainThread:5872 [wandb_init.py:init():857] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-06-26 14:40:00,444 INFO    MainThread:5872 [wandb_init.py:init():893] starting backend
+2025-06-26 14:40:00,444 INFO    MainThread:5872 [wandb_init.py:init():897] sending inform_init request
+2025-06-26 14:40:00,446 INFO    MainThread:5872 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-06-26 14:40:00,452 INFO    MainThread:5872 [wandb_init.py:init():907] backend started and connected
+2025-06-26 14:40:00,460 INFO    MainThread:5872 [wandb_init.py:init():1005] updated telemetry
+2025-06-26 14:40:00,464 INFO    MainThread:5872 [wandb_init.py:init():1029] communicating run to backend with 90.0 second timeout
+2025-06-26 14:40:05,603 INFO    MainThread:5872 [wandb_init.py:init():1104] starting run threads in backend
+2025-06-26 14:40:05,851 INFO    MainThread:5872 [wandb_run.py:_console_start():2573] atexit reg
+2025-06-26 14:40:05,851 INFO    MainThread:5872 [wandb_run.py:_redirect():2421] redirect: wrap_raw
+2025-06-26 14:40:05,855 INFO    MainThread:5872 [wandb_run.py:_redirect():2490] Wrapping output streams.
+2025-06-26 14:40:05,855 INFO    MainThread:5872 [wandb_run.py:_redirect():2513] Redirects installed.
+2025-06-26 14:40:05,857 INFO    MainThread:5872 [wandb_init.py:init():1150] run started, returning control to user process
+2025-06-26 14:40:11,773 INFO    MainThread:5872 [wandb_run.py:_config_callback():1436] config_cb None None {'filename': 'stage1_06261435', 'seed': 42, 'mode': 'train', 'strategy': 'deepspeed', 'accelerator': 'gpu', 'devices': '0,1,2,3,4,5,6,7', 'precision': 'bf16-mixed', 'max_epochs': 30, 'check_val_every_n_epoch': 1, 'use_wandb_logger': True, 'mix_dataset': True, 'temperature': 0.1, 'save_every_n_epochs': 5, 'ptm': True, 'lm': True, 'rerank_cand_num': 128, 'plm_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m', 'plm_tune': 'freeze', 'load_4bit': False, 'pool_size': 0, 'bert_hidden_dim': 768, 'bert_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft', 'projection_dim': 256, 'cross_attention_freq': 2, 'num_query_token': 8, 'weight_decay': 0.05, 'init_lr': 0.0001, 'min_lr': 1e-05, 'warmup_lr': 1e-06, 'warmup_steps': 1000, 'lr_decay_rate': 0.9, 'scheduler': 'linear_warmup_cosine_lr', 'init_checkpoint': '', 'retrieval_eval_epoch': 10, 'num_workers': 8, 'batch_size': 64, 'match_batch_size': 64, 'root': 'data', 'text_max_len': 128, 'prot_max_len': 1024, 'prot_aug': 'None'}
+2025-06-26 14:41:27,870 INFO    MsgRouterThr:5872 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles.

ProtT3/all_checkpoints/stage1_06261435/wandb/run-20250626_144731-1bz1vyyf/files/output.log

@@ -0,0 +1,28 @@
+/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:654: Checkpoint directory /nas/shared/kilab/wangyujia/ProtT3/all_checkpoints/stage1_06261435 exists and is not empty.
+Enabling DeepSpeed BF16. Model parameters and inputs will be cast to `bfloat16`.
+LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]
+
+  | Name         | Type         | Params | Mode
+------------------------------------------------------
+0 | blip2qformer | Blip2Qformer | 327 M  | train
+------------------------------------------------------
+179 M     Trainable params
+147 M     Non-trainable params
+327 M     Total params
+1,309.467 Total estimated model params size (MB)
+5         Modules in train mode
+926       Modules in eval mode
+Epoch 9: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 666/666 [31:47<00:00,  0.35it/s, v_num=vyyf]
+/nas/shared/kilab/wangyujia/ProtT3/model/blip2qformer.py:220: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(enable_autocast, dtype=torch.float32):
+Validation DataLoader 2: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 8/8 [00:18<00:00,  0.44it/s]
+/nas/shared/kilab/wangyujia/ProtT3/model/dist_funs.py:18: FutureWarning: Positional args are being deprecated, use kwargs instead. Refer to https://pytorch.org/docs/main/generated/torch.nn.Module.html#torch.nn.Module.state_dict for details.
+  sd = self.module.state_dict(destination, prefix, keep_vars)
+/nas/shared/kilab/wangyujia/ProtT3/model/blip2_stage1.py:42: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  return torch.cuda.amp.autocast(dtype=dtype)
+100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 157/157 [08:51<00:00,  3.39s/it]
+100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1250/1250 [00:01<00:00, 1063.25it/s]
+re-ranking p2t: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████���███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2500/2500 [10:06<00:00,  4.12it/s]
+ 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍              | 1179/1250 [00:01<00:00, 1084.47it/s][rank: 1] Child process with PID 14146 terminated with code -6. Forcefully terminating all other processes to avoid zombies 🧟
+re-ranking p2t: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2500/2500 [10:06<00:00,  4.13it/s]
+re-ranking t2p:  58%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍                                                                                                     | 1451/2500 [10:59<08:00,  2.18it/s]

ProtT3/all_checkpoints/stage1_06261435/wandb/run-20250626_144731-1bz1vyyf/files/requirements.txt

@@ -0,0 +1,225 @@
+attrs==25.3.0
+tqdm==4.67.1
+langcodes==3.5.0
+nvidia-cublas-cu12==12.4.5.8
+tifffile==2025.5.10
+nvidia-cufile-cu12==1.11.1.6
+nltk==3.9.1
+salesforce-lavis==1.0.2
+tzdata==2025.2
+pyparsing==3.2.3
+six==1.17.0
+python-dateutil==2.9.0.post0
+pandas==2.2.3
+pytorch-lightning==2.5.1.post0
+blinker==1.9.0
+opencv-python-headless==4.5.5.64
+nvidia-cusparse-cu12==12.3.1.170
+pytz==2025.2
+async-timeout==5.0.1
+pillow==11.2.1
+parso==0.8.4
+joblib==1.5.1
+contourpy==1.3.2
+triton==3.2.0
+marisa-trie==1.2.1
+PyYAML==6.0.2
+regex==2024.11.6
+idna==3.10
+nvidia-curand-cu12==10.3.5.147
+rpds-py==0.25.1
+aiosignal==1.3.2
+srsly==2.5.1
+confection==0.1.5
+typing-inspection==0.4.1
+packaging==24.2
+distlib==0.3.9
+networkx==3.4.2
+absl-py==2.2.2
+yarl==1.20.0
+lightning-utilities==0.14.3
+executing==2.2.0
+pycocoevalcap==1.2
+wheel==0.45.1
+nvidia-ml-py==12.575.51
+cycler==0.12.1
+wrapt==1.17.2
+jsonschema-specifications==2025.4.1
+protobuf==6.31.0
+mpmath==1.3.0
+certifi==2025.4.26
+py-cpuinfo==9.0.0
+contexttimer==0.3.3
+watchdog==6.0.0
+pexpect==4.9.0
+webencodings==0.5.1
+hf-xet==1.1.2
+cymem==2.0.11
+requests==2.32.3
+timm==0.4.12
+omegaconf==2.3.0
+nvidia-nvjitlink-cu12==12.4.127
+webdataset==0.2.111
+nodeenv==1.9.1
+frozenlist==1.6.0
+annotated-types==0.7.0
+matplotlib-inline==0.1.7
+urllib3==2.4.0
+rich==14.0.0
+GitPython==3.1.44
+lazy_loader==0.4
+msgpack==1.1.0
+prompt_toolkit==3.0.51
+fonttools==4.58.0
+multidict==6.4.4
+blis==1.3.0
+thinc==8.3.6
+nvidia-nvtx-cu12==12.4.127
+torchmetrics==1.7.1
+weasel==0.4.1
+numpy==2.2.6
+cachetools==5.5.2
+Jinja2==3.1.6
+matplotlib==3.10.3
+nvidia-cudnn-cu12==9.1.0.70
+Pygments==2.19.1
+tornado==6.5.1
+scipy==1.15.3
+rouge_score==0.1.2
+cloudpathlib==0.21.1
+jedi==0.19.2
+referencing==0.36.2
+decord==0.6.0
+setuptools==78.1.1
+mdurl==0.1.2
+identify==2.6.12
+python-slugify==8.0.4
+portalocker==3.1.1
+catalogue==2.0.10
+platformdirs==4.3.8
+antlr4-python3-runtime==4.9.3
+nvidia-cusolver-cu12==11.6.1.9
+kaggle==1.7.4.5
+pydeck==0.9.1
+pydantic==2.11.5
+nvidia-cufft-cu12==11.2.1.3
+pyarrow==20.0.0
+nvidia-nccl-cu12==2.21.5
+markdown-it-py==3.0.0
+gitdb==4.0.12
+altair==5.5.0
+torchvision==0.21.0
+python-magic==0.4.27
+iopath==0.1.10
+smart-open==7.1.0
+torch==2.6.0
+pycocotools==2.0.8
+fairscale==0.4.4
+traitlets==5.14.3
+pure_eval==0.2.3
+sympy==1.13.1
+nvidia-cusparselt-cu12==0.6.2
+imageio==2.37.0
+stack-data==0.6.3
+shellingham==1.5.4
+nvidia-cuda-runtime-cu12==12.4.127
+einops==0.8.1
+tenacity==9.1.2
+virtualenv==20.31.2
+ptyprocess==0.7.0
+cfgv==3.4.0
+pre_commit==4.2.0
+language_data==1.3.0
+typing_extensions==4.13.2
+propcache==0.3.1
+nvidia-cuda-cupti-cu12==12.4.127
+safetensors==0.5.3
+text-unidecode==1.3
+wcwidth==0.2.13
+charset-normalizer==3.4.2
+aiohappyeyeballs==2.6.1
+ipython==8.36.0
+streamlit==1.45.1
+asttokens==3.0.0
+psutil==7.0.0
+smmap==5.0.2
+exceptiongroup==1.3.0
+murmurhash==1.0.13
+filelock==3.18.0
+plotly==6.1.1
+hjson==3.1.0
+pydantic_core==2.33.2
+ninja==1.11.1.4
+kiwisolver==1.4.8
+spacy-legacy==3.0.12
+opendatasets==0.1.22
+decorator==5.2.1
+spacy==3.8.7
+wasabi==1.1.3
+sentencepiece==0.2.0
+toml==0.10.2
+scikit-image==0.25.2
+deepspeed==0.16.10+b666844f
+ftfy==6.3.1
+bleach==6.2.0
+nvidia-cuda-nvrtc-cu12==12.4.127
+spacy-loggers==1.0.5
+MarkupSafe==3.0.2
+braceexpand==0.1.7
+oss2==2.15.0
+preshed==3.0.10
+transformers==4.52.3
+aiohttp==3.12.2
+web.py==0.62
+threadpoolctl==3.6.0
+jaraco.functools==4.1.0
+wandb==0.19.11
+sentry-sdk==2.29.1
+tokenizers==0.21.1
+fsspec==2025.3.0
+flash-attn==2.7.1.post1
+opendelta==0.3.2
+opencv-python==4.11.0.86
+click==8.2.1
+docker-pycreds==0.4.0
+typer==0.16.0
+xxhash==3.5.0
+pathlib==1.0.1
+dill==0.3.8
+crcmod==1.7
+bigmodelvis==0.0.1
+datasets==3.6.0
+pycryptodome==3.23.0
+jsonschema==4.24.0
+aliyun-python-sdk-core==2.16.0
+jmespath==0.10.0
+more-itertools==10.7.0
+scikit-learn==1.6.1
+huggingface-hub==0.32.1
+cryptography==45.0.3
+pycparser==2.22
+yacs==0.1.8
+aliyun-python-sdk-kms==2.16.5
+cffi==1.17.1
+delta-center-client==0.0.4
+multiprocess==0.70.16
+setproctitle==1.3.6
+narwhals==1.41.0
+pip==25.1.1
+cheroot==10.0.1
+jaraco.context==5.3.0
+more-itertools==10.3.0
+jaraco.functools==4.0.1
+jaraco.text==3.12.1
+platformdirs==4.2.2
+packaging==24.2
+wheel==0.45.1
+zipp==3.19.2
+inflect==7.3.1
+autocommand==2.2.2
+typeguard==4.3.0
+jaraco.collections==5.1.0
+backports.tarfile==1.2.0
+tomli==2.0.1
+importlib_metadata==8.0.0
+typing_extensions==4.12.2

ProtT3/all_checkpoints/stage1_06261435/wandb/run-20250626_144731-1bz1vyyf/files/wandb-metadata.json

@@ -0,0 +1,98 @@
+{
+  "os": "Linux-5.10.134-008.16.kangaroo.al8.x86_64-x86_64-with-glibc2.35",
+  "python": "CPython 3.10.0",
+  "startedAt": "2025-06-26T06:47:31.299461Z",
+  "args": [
+    "--devices",
+    "0,1,2,3,4,5,6,7",
+    "--mode",
+    "train",
+    "--filename",
+    "stage1_06261435",
+    "--num_query_token",
+    "8",
+    "--plm_name",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m",
+    "--bert_name",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft",
+    "--save_every_n_epochs",
+    "5",
+    "--max_epochs",
+    "30",
+    "--batch_size",
+    "160",
+    "--precision",
+    "bf16-mixed",
+    "--mix_dataset",
+    "--num_workers",
+    "8",
+    "--use_wandb_logger"
+  ],
+  "program": "/nas/shared/kilab/wangyujia/ProtT3/stage1.py",
+  "codePath": "stage1.py",
+  "email": "gia0603yucca@gmail.com",
+  "root": "./all_checkpoints/stage1_06261435/",
+  "host": "dsw-265304-cd576ddc5-gh74w",
+  "executable": "/root/miniconda3/envs/protT3/bin/python",
+  "codePathLocal": "stage1.py",
+  "cpu_count": 64,
+  "cpu_count_logical": 64,
+  "gpu": "NVIDIA A800-SXM4-80GB",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1623302262784",
+      "used": "1290838016"
+    }
+  },
+  "memory": {
+    "total": "549755813888"
+  },
+  "cpu": {
+    "count": 64,
+    "countLogical": 64
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    }
+  ],
+  "cudaVersion": "12.1"
+}

ProtT3/all_checkpoints/stage1_06261435/wandb/run-20250626_144731-1bz1vyyf/logs/debug-internal.log

@@ -0,0 +1,15 @@
+{"time":"2025-06-26T14:47:31.30118788+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.11","symlink path":"all_checkpoints/stage1_06261435/wandb/run-20250626_144731-1bz1vyyf/logs/debug-core.log"}
+{"time":"2025-06-26T14:47:33.763139479+08:00","level":"INFO","msg":"created new stream","id":"1bz1vyyf"}
+{"time":"2025-06-26T14:47:33.763180996+08:00","level":"INFO","msg":"stream: started","id":"1bz1vyyf"}
+{"time":"2025-06-26T14:47:33.76320552+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"1bz1vyyf"}
+{"time":"2025-06-26T14:47:33.763263895+08:00","level":"INFO","msg":"handler: started","stream_id":"1bz1vyyf"}
+{"time":"2025-06-26T14:47:33.763302435+08:00","level":"INFO","msg":"sender: started","stream_id":"1bz1vyyf"}
+{"time":"2025-06-26T14:47:35.049823143+08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2025-06-26T14:59:24.18296941+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06261435/1bz1vyyf/file_stream\": EOF"}
+{"time":"2025-06-26T15:47:20.217171547+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-26T15:47:52.39492821+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-26T15:48:27.272812301+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-06-26T15:49:05.914803533+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-26T15:49:51.452193247+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06261435/1bz1vyyf/file_stream\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-26T15:49:52.900302871+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-06-26T15:50:55.486728897+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}

ProtT3/all_checkpoints/stage1_06261435/wandb/run-20250626_144731-1bz1vyyf/logs/debug.log

@@ -0,0 +1,23 @@
+2025-06-26 14:47:31,292 INFO    MainThread:13641 [wandb_setup.py:_flush():70] Current SDK version is 0.19.11
+2025-06-26 14:47:31,292 INFO    MainThread:13641 [wandb_setup.py:_flush():70] Configure stats pid to 13641
+2025-06-26 14:47:31,292 INFO    MainThread:13641 [wandb_setup.py:_flush():70] Loading settings from /root/.config/wandb/settings
+2025-06-26 14:47:31,292 INFO    MainThread:13641 [wandb_setup.py:_flush():70] Loading settings from /nas/shared/kilab/wangyujia/ProtT3/wandb/settings
+2025-06-26 14:47:31,292 INFO    MainThread:13641 [wandb_setup.py:_flush():70] Loading settings from environment variables
+2025-06-26 14:47:31,292 INFO    MainThread:13641 [wandb_init.py:setup_run_log_directory():724] Logging user logs to ./all_checkpoints/stage1_06261435/wandb/run-20250626_144731-1bz1vyyf/logs/debug.log
+2025-06-26 14:47:31,292 INFO    MainThread:13641 [wandb_init.py:setup_run_log_directory():725] Logging internal logs to ./all_checkpoints/stage1_06261435/wandb/run-20250626_144731-1bz1vyyf/logs/debug-internal.log
+2025-06-26 14:47:31,292 INFO    MainThread:13641 [wandb_init.py:init():852] calling init triggers
+2025-06-26 14:47:31,292 INFO    MainThread:13641 [wandb_init.py:init():857] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-06-26 14:47:31,292 INFO    MainThread:13641 [wandb_init.py:init():893] starting backend
+2025-06-26 14:47:31,292 INFO    MainThread:13641 [wandb_init.py:init():897] sending inform_init request
+2025-06-26 14:47:31,294 INFO    MainThread:13641 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-06-26 14:47:31,296 INFO    MainThread:13641 [wandb_init.py:init():907] backend started and connected
+2025-06-26 14:47:31,300 INFO    MainThread:13641 [wandb_init.py:init():1005] updated telemetry
+2025-06-26 14:47:31,303 INFO    MainThread:13641 [wandb_init.py:init():1029] communicating run to backend with 90.0 second timeout
+2025-06-26 14:47:35,041 INFO    MainThread:13641 [wandb_init.py:init():1104] starting run threads in backend
+2025-06-26 14:47:35,187 INFO    MainThread:13641 [wandb_run.py:_console_start():2573] atexit reg
+2025-06-26 14:47:35,187 INFO    MainThread:13641 [wandb_run.py:_redirect():2421] redirect: wrap_raw
+2025-06-26 14:47:35,192 INFO    MainThread:13641 [wandb_run.py:_redirect():2490] Wrapping output streams.
+2025-06-26 14:47:35,192 INFO    MainThread:13641 [wandb_run.py:_redirect():2513] Redirects installed.
+2025-06-26 14:47:35,197 INFO    MainThread:13641 [wandb_init.py:init():1150] run started, returning control to user process
+2025-06-26 14:47:41,914 INFO    MainThread:13641 [wandb_run.py:_config_callback():1436] config_cb None None {'filename': 'stage1_06261435', 'seed': 42, 'mode': 'train', 'strategy': 'deepspeed', 'accelerator': 'gpu', 'devices': '0,1,2,3,4,5,6,7', 'precision': 'bf16-mixed', 'max_epochs': 30, 'check_val_every_n_epoch': 1, 'use_wandb_logger': True, 'mix_dataset': True, 'temperature': 0.1, 'save_every_n_epochs': 5, 'ptm': True, 'lm': True, 'rerank_cand_num': 128, 'plm_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m', 'plm_tune': 'freeze', 'load_4bit': False, 'pool_size': 0, 'bert_hidden_dim': 768, 'bert_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft', 'projection_dim': 256, 'cross_attention_freq': 2, 'num_query_token': 8, 'weight_decay': 0.05, 'init_lr': 0.0001, 'min_lr': 1e-05, 'warmup_lr': 1e-06, 'warmup_steps': 1000, 'lr_decay_rate': 0.9, 'scheduler': 'linear_warmup_cosine_lr', 'init_checkpoint': '', 'retrieval_eval_epoch': 10, 'num_workers': 8, 'batch_size': 160, 'match_batch_size': 64, 'root': 'data', 'text_max_len': 128, 'prot_max_len': 1024, 'prot_aug': 'None'}

ProtT3/all_checkpoints/stage1_06262112/wandb/debug-internal.log

@@ -0,0 +1,73 @@
+{"time":"2025-06-26T21:13:59.919018005+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.11","symlink path":"all_checkpoints/stage1_06262112/wandb/run-20250626_211359-gp8ndo2v/logs/debug-core.log"}
+{"time":"2025-06-26T21:14:01.751242877+08:00","level":"INFO","msg":"created new stream","id":"gp8ndo2v"}
+{"time":"2025-06-26T21:14:01.751292945+08:00","level":"INFO","msg":"stream: started","id":"gp8ndo2v"}
+{"time":"2025-06-26T21:14:01.751353982+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"gp8ndo2v"}
+{"time":"2025-06-26T21:14:01.751376676+08:00","level":"INFO","msg":"handler: started","stream_id":"gp8ndo2v"}
+{"time":"2025-06-26T21:14:01.751406784+08:00","level":"INFO","msg":"sender: started","stream_id":"gp8ndo2v"}
+{"time":"2025-06-26T21:14:03.18201785+08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2025-06-26T21:39:10.805194559+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:48308->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-06-26T21:45:08.181153524+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:33626->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-26T21:50:43.748258238+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:39240->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-06-26T21:56:59.349224169+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:41628->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-26T22:01:29.173164681+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:41150->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-06-26T22:04:52.9491833+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:40328->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-26T22:05:33.372515641+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-26T22:10:24.214205918+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:39814->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-26T22:11:19.608808233+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": unexpected EOF"}
+{"time":"2025-06-26T22:15:15.541207766+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:57490->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-26T22:17:53.749178371+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:34226->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-06-26T22:20:43.734188539+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:58158->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-26T22:26:37.244674658+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-26T22:27:27.97084057+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:42482->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-06-26T22:32:12.373221258+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:42718->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-26T22:33:45.749714178+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:36352->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-06-26T22:34:27.154183486+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": unexpected EOF"}
+{"time":"2025-06-26T22:37:03.388715023+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-26T22:38:44.053145624+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:56120->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-26T22:39:26.21620593+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:37762->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-06-26T22:42:48.392517517+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-26T22:43:20.509939526+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-06-26T22:43:55.50812991+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-06-26T22:44:05.260626832+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": context deadline exceeded"}
+{"time":"2025-06-26T22:44:33.704733361+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-26T22:47:28.34118454+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:46752->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-26T22:48:22.800067638+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:35900->104.21.20.172:443: read: connection reset by peer"}
+{"time":"2025-06-26T22:49:03.396821287+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-26T22:49:35.881823651+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-06-26T22:50:10.244289946+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-06-26T22:50:48.344767175+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-06-26T22:51:21.302223032+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:40288->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-06-26T22:54:35.861164416+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:35106->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-26T22:56:18.401507947+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-26T22:56:50.726216671+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-06-26T22:57:25.218970516+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-26T22:58:04.180971507+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-06-26T22:58:09.302334148+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:50854->104.21.20.172:443: read: connection reset by peer"}
+{"time":"2025-06-26T22:58:51.546103299+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-26T23:00:01.390576029+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-26T23:02:34.06917085+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:37644->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-26T23:05:03.407063467+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-26T23:06:15.25317264+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:42244->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-06-26T23:06:33.40769986+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-06-26T23:09:18.410102964+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-26T23:11:33.411287647+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-26T23:12:46.933205687+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:55392->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-26T23:16:18.41464022+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-26T23:16:40.753618008+08:00","level":"ERROR","msg":"filestream: json decode error: net/http: request canceled (Client.Timeout or context cancellation while reading body)"}
+{"time":"2025-06-26T23:16:50.454824576+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-06-26T23:17:25.158378302+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-26T23:18:00.15504283+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": unexpected EOF"}
+{"time":"2025-06-26T23:18:03.880963877+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-26T23:18:50.836175421+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-26T23:20:45.653178557+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:38816->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-26T23:21:51.085821178+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:43680->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-06-26T23:24:50.901213106+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:40734->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-26T23:30:06.293188303+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:46360->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-26T23:36:26.709172933+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:33352->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-06-26T23:39:53.889169333+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": unexpected EOF"}
+{"time":"2025-06-26T23:42:48.781078984+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": read tcp 10.1.8.160:35500->104.21.20.172:443: read: connection reset by peer"}
+{"time":"2025-06-26T23:44:39.969525919+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:51204->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-06-26T23:52:25.685198314+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:41504->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-06-27T00:14:48.690187795+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": read tcp 10.1.8.160:49576->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-06-27T00:15:11.63645902+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": unexpected EOF"}
+{"time":"2025-06-27T00:18:18.739690809+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": read tcp 10.1.8.160:55462->172.67.193.61:443: read: connection reset by peer"}

ProtT3/all_checkpoints/stage1_06262112/wandb/debug.log

@@ -0,0 +1,23 @@
+2025-06-26 21:13:59,905 INFO    MainThread:183028 [wandb_setup.py:_flush():70] Current SDK version is 0.19.11
+2025-06-26 21:13:59,905 INFO    MainThread:183028 [wandb_setup.py:_flush():70] Configure stats pid to 183028
+2025-06-26 21:13:59,905 INFO    MainThread:183028 [wandb_setup.py:_flush():70] Loading settings from /root/.config/wandb/settings
+2025-06-26 21:13:59,905 INFO    MainThread:183028 [wandb_setup.py:_flush():70] Loading settings from /nas/shared/kilab/wangyujia/ProtT3/wandb/settings
+2025-06-26 21:13:59,905 INFO    MainThread:183028 [wandb_setup.py:_flush():70] Loading settings from environment variables
+2025-06-26 21:13:59,905 INFO    MainThread:183028 [wandb_init.py:setup_run_log_directory():724] Logging user logs to ./all_checkpoints/stage1_06262112/wandb/run-20250626_211359-gp8ndo2v/logs/debug.log
+2025-06-26 21:13:59,905 INFO    MainThread:183028 [wandb_init.py:setup_run_log_directory():725] Logging internal logs to ./all_checkpoints/stage1_06262112/wandb/run-20250626_211359-gp8ndo2v/logs/debug-internal.log
+2025-06-26 21:13:59,905 INFO    MainThread:183028 [wandb_init.py:init():852] calling init triggers
+2025-06-26 21:13:59,905 INFO    MainThread:183028 [wandb_init.py:init():857] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-06-26 21:13:59,905 INFO    MainThread:183028 [wandb_init.py:init():893] starting backend
+2025-06-26 21:13:59,905 INFO    MainThread:183028 [wandb_init.py:init():897] sending inform_init request
+2025-06-26 21:13:59,906 INFO    MainThread:183028 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-06-26 21:13:59,908 INFO    MainThread:183028 [wandb_init.py:init():907] backend started and connected
+2025-06-26 21:13:59,909 INFO    MainThread:183028 [wandb_init.py:init():1005] updated telemetry
+2025-06-26 21:13:59,911 INFO    MainThread:183028 [wandb_init.py:init():1029] communicating run to backend with 90.0 second timeout
+2025-06-26 21:14:03,171 INFO    MainThread:183028 [wandb_init.py:init():1104] starting run threads in backend
+2025-06-26 21:14:03,332 INFO    MainThread:183028 [wandb_run.py:_console_start():2573] atexit reg
+2025-06-26 21:14:03,333 INFO    MainThread:183028 [wandb_run.py:_redirect():2421] redirect: wrap_raw
+2025-06-26 21:14:03,336 INFO    MainThread:183028 [wandb_run.py:_redirect():2490] Wrapping output streams.
+2025-06-26 21:14:03,336 INFO    MainThread:183028 [wandb_run.py:_redirect():2513] Redirects installed.
+2025-06-26 21:14:03,350 INFO    MainThread:183028 [wandb_init.py:init():1150] run started, returning control to user process
+2025-06-26 21:14:09,516 INFO    MainThread:183028 [wandb_run.py:_config_callback():1436] config_cb None None {'filename': 'stage1_06262112', 'seed': 42, 'mode': 'train', 'strategy': 'deepspeed', 'accelerator': 'gpu', 'devices': '0,1,2,3,4,5,6,7', 'precision': 'bf16-mixed', 'max_epochs': 30, 'check_val_every_n_epoch': 1, 'use_wandb_logger': True, 'mix_dataset': True, 'temperature': 0.1, 'save_every_n_epochs': 5, 'ptm': True, 'lm': True, 'rerank_cand_num': 128, 'plm_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m', 'plm_tune': 'freeze', 'load_4bit': False, 'pool_size': 0, 'bert_hidden_dim': 768, 'bert_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft', 'projection_dim': 256, 'cross_attention_freq': 2, 'num_query_token': 8, 'weight_decay': 0.05, 'init_lr': 0.0001, 'min_lr': 1e-05, 'warmup_lr': 1e-06, 'warmup_steps': 1000, 'lr_decay_rate': 0.9, 'scheduler': 'linear_warmup_cosine_lr', 'init_checkpoint': '', 'retrieval_eval_epoch': 10, 'num_workers': 8, 'batch_size': 160, 'match_batch_size': 64, 'root': 'data', 'text_max_len': 128, 'prot_max_len': 1024, 'prot_aug': 'None'}

ProtT3/all_checkpoints/stage1_06262112/wandb/run-20250626_211359-gp8ndo2v/files/output.log

@@ -0,0 +1,28 @@
+/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:654: Checkpoint directory /nas/shared/kilab/wangyujia/ProtT3/all_checkpoints/stage1_06262112 exists and is not empty.
+Enabling DeepSpeed BF16. Model parameters and inputs will be cast to `bfloat16`.
+LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]
+
+  | Name         | Type         | Params | Mode
+------------------------------------------------------
+0 | blip2qformer | Blip2Qformer | 327 M  | train
+------------------------------------------------------
+179 M     Trainable params
+147 M     Non-trainable params
+327 M     Total params
+1,309.467 Total estimated model params size (MB)
+5         Modules in train mode
+926       Modules in eval mode
+Epoch 9: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 336/336 [16:02<00:00,  0.35it/s, v_num=do2v]
+/nas/shared/kilab/wangyujia/ProtT3/model/blip2qformer.py:220: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(enable_autocast, dtype=torch.float32):
+Validation DataLoader 2: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 8/8 [00:18<00:00,  0.44it/s]
+/nas/shared/kilab/wangyujia/ProtT3/model/dist_funs.py:18: FutureWarning: Positional args are being deprecated, use kwargs instead. Refer to https://pytorch.org/docs/main/generated/torch.nn.Module.html#torch.nn.Module.state_dict for details.
+  sd = self.module.state_dict(destination, prefix, keep_vars)
+/nas/shared/kilab/wangyujia/ProtT3/model/blip2_stage1.py:42: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  return torch.cuda.amp.autocast(dtype=dtype)
+100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 157/157 [08:52<00:00,  3.39s/it]
+100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1250/1250 [00:01<00:00, 1024.41it/s]
+re-ranking p2t: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████���███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2500/2500 [10:06<00:00,  4.12it/s]
+ 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████               | 1177/1250 [00:01<00:00, 1045.73it/s][rank: 1] Child process with PID 183538 terminated with code -6. Forcefully terminating all other processes to avoid zombies 🧟
+re-ranking p2t: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2500/2500 [10:06<00:00,  4.13it/s]
+re-ranking t2p:  57%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊                                                                                                        | 1424/2500 [11:01<08:20,  2.15it/s]

ProtT3/all_checkpoints/stage1_06262112/wandb/run-20250626_211359-gp8ndo2v/files/requirements.txt

@@ -0,0 +1,225 @@
+attrs==25.3.0
+tqdm==4.67.1
+langcodes==3.5.0
+nvidia-cublas-cu12==12.4.5.8
+tifffile==2025.5.10
+nvidia-cufile-cu12==1.11.1.6
+nltk==3.9.1
+salesforce-lavis==1.0.2
+tzdata==2025.2
+pyparsing==3.2.3
+six==1.17.0
+python-dateutil==2.9.0.post0
+pandas==2.2.3
+pytorch-lightning==2.5.1.post0
+blinker==1.9.0
+opencv-python-headless==4.5.5.64
+nvidia-cusparse-cu12==12.3.1.170
+pytz==2025.2
+async-timeout==5.0.1
+pillow==11.2.1
+parso==0.8.4
+joblib==1.5.1
+contourpy==1.3.2
+triton==3.2.0
+marisa-trie==1.2.1
+PyYAML==6.0.2
+regex==2024.11.6
+idna==3.10
+nvidia-curand-cu12==10.3.5.147
+rpds-py==0.25.1
+aiosignal==1.3.2
+srsly==2.5.1
+confection==0.1.5
+typing-inspection==0.4.1
+packaging==24.2
+distlib==0.3.9
+networkx==3.4.2
+absl-py==2.2.2
+yarl==1.20.0
+lightning-utilities==0.14.3
+executing==2.2.0
+pycocoevalcap==1.2
+wheel==0.45.1
+nvidia-ml-py==12.575.51
+cycler==0.12.1
+wrapt==1.17.2
+jsonschema-specifications==2025.4.1
+protobuf==6.31.0
+mpmath==1.3.0
+certifi==2025.4.26
+py-cpuinfo==9.0.0
+contexttimer==0.3.3
+watchdog==6.0.0
+pexpect==4.9.0
+webencodings==0.5.1
+hf-xet==1.1.2
+cymem==2.0.11
+requests==2.32.3
+timm==0.4.12
+omegaconf==2.3.0
+nvidia-nvjitlink-cu12==12.4.127
+webdataset==0.2.111
+nodeenv==1.9.1
+frozenlist==1.6.0
+annotated-types==0.7.0
+matplotlib-inline==0.1.7
+urllib3==2.4.0
+rich==14.0.0
+GitPython==3.1.44
+lazy_loader==0.4
+msgpack==1.1.0
+prompt_toolkit==3.0.51
+fonttools==4.58.0
+multidict==6.4.4
+blis==1.3.0
+thinc==8.3.6
+nvidia-nvtx-cu12==12.4.127
+torchmetrics==1.7.1
+weasel==0.4.1
+numpy==2.2.6
+cachetools==5.5.2
+Jinja2==3.1.6
+matplotlib==3.10.3
+nvidia-cudnn-cu12==9.1.0.70
+Pygments==2.19.1
+tornado==6.5.1
+scipy==1.15.3
+rouge_score==0.1.2
+cloudpathlib==0.21.1
+jedi==0.19.2
+referencing==0.36.2
+decord==0.6.0
+setuptools==78.1.1
+mdurl==0.1.2
+identify==2.6.12
+python-slugify==8.0.4
+portalocker==3.1.1
+catalogue==2.0.10
+platformdirs==4.3.8
+antlr4-python3-runtime==4.9.3
+nvidia-cusolver-cu12==11.6.1.9
+kaggle==1.7.4.5
+pydeck==0.9.1
+pydantic==2.11.5
+nvidia-cufft-cu12==11.2.1.3
+pyarrow==20.0.0
+nvidia-nccl-cu12==2.21.5
+markdown-it-py==3.0.0
+gitdb==4.0.12
+altair==5.5.0
+torchvision==0.21.0
+python-magic==0.4.27
+iopath==0.1.10
+smart-open==7.1.0
+torch==2.6.0
+pycocotools==2.0.8
+fairscale==0.4.4
+traitlets==5.14.3
+pure_eval==0.2.3
+sympy==1.13.1
+nvidia-cusparselt-cu12==0.6.2
+imageio==2.37.0
+stack-data==0.6.3
+shellingham==1.5.4
+nvidia-cuda-runtime-cu12==12.4.127
+einops==0.8.1
+tenacity==9.1.2
+virtualenv==20.31.2
+ptyprocess==0.7.0
+cfgv==3.4.0
+pre_commit==4.2.0
+language_data==1.3.0
+typing_extensions==4.13.2
+propcache==0.3.1
+nvidia-cuda-cupti-cu12==12.4.127
+safetensors==0.5.3
+text-unidecode==1.3
+wcwidth==0.2.13
+charset-normalizer==3.4.2
+aiohappyeyeballs==2.6.1
+ipython==8.36.0
+streamlit==1.45.1
+asttokens==3.0.0
+psutil==7.0.0
+smmap==5.0.2
+exceptiongroup==1.3.0
+murmurhash==1.0.13
+filelock==3.18.0
+plotly==6.1.1
+hjson==3.1.0
+pydantic_core==2.33.2
+ninja==1.11.1.4
+kiwisolver==1.4.8
+spacy-legacy==3.0.12
+opendatasets==0.1.22
+decorator==5.2.1
+spacy==3.8.7
+wasabi==1.1.3
+sentencepiece==0.2.0
+toml==0.10.2
+scikit-image==0.25.2
+deepspeed==0.16.10+b666844f
+ftfy==6.3.1
+bleach==6.2.0
+nvidia-cuda-nvrtc-cu12==12.4.127
+spacy-loggers==1.0.5
+MarkupSafe==3.0.2
+braceexpand==0.1.7
+oss2==2.15.0
+preshed==3.0.10
+transformers==4.52.3
+aiohttp==3.12.2
+web.py==0.62
+threadpoolctl==3.6.0
+jaraco.functools==4.1.0
+wandb==0.19.11
+sentry-sdk==2.29.1
+tokenizers==0.21.1
+fsspec==2025.3.0
+flash-attn==2.7.1.post1
+opendelta==0.3.2
+opencv-python==4.11.0.86
+click==8.2.1
+docker-pycreds==0.4.0
+typer==0.16.0
+xxhash==3.5.0
+pathlib==1.0.1
+dill==0.3.8
+crcmod==1.7
+bigmodelvis==0.0.1
+datasets==3.6.0
+pycryptodome==3.23.0
+jsonschema==4.24.0
+aliyun-python-sdk-core==2.16.0
+jmespath==0.10.0
+more-itertools==10.7.0
+scikit-learn==1.6.1
+huggingface-hub==0.32.1
+cryptography==45.0.3
+pycparser==2.22
+yacs==0.1.8
+aliyun-python-sdk-kms==2.16.5
+cffi==1.17.1
+delta-center-client==0.0.4
+multiprocess==0.70.16
+setproctitle==1.3.6
+narwhals==1.41.0
+pip==25.1.1
+cheroot==10.0.1
+jaraco.context==5.3.0
+more-itertools==10.3.0
+jaraco.functools==4.0.1
+jaraco.text==3.12.1
+platformdirs==4.2.2
+packaging==24.2
+wheel==0.45.1
+zipp==3.19.2
+inflect==7.3.1
+autocommand==2.2.2
+typeguard==4.3.0
+jaraco.collections==5.1.0
+backports.tarfile==1.2.0
+tomli==2.0.1
+importlib_metadata==8.0.0
+typing_extensions==4.12.2

ProtT3/all_checkpoints/stage1_06262112/wandb/run-20250626_211359-gp8ndo2v/files/wandb-metadata.json

@@ -0,0 +1,98 @@
+{
+  "os": "Linux-5.10.134-008.16.kangaroo.al8.x86_64-x86_64-with-glibc2.35",
+  "python": "CPython 3.10.0",
+  "startedAt": "2025-06-26T13:13:59.908975Z",
+  "args": [
+    "--devices",
+    "0,1,2,3,4,5,6,7",
+    "--mode",
+    "train",
+    "--filename",
+    "stage1_06262112",
+    "--num_query_token",
+    "8",
+    "--plm_name",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m",
+    "--bert_name",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft",
+    "--save_every_n_epochs",
+    "5",
+    "--max_epochs",
+    "30",
+    "--batch_size",
+    "160",
+    "--precision",
+    "bf16-mixed",
+    "--mix_dataset",
+    "--num_workers",
+    "8",
+    "--use_wandb_logger"
+  ],
+  "program": "/nas/shared/kilab/wangyujia/ProtT3/stage1.py",
+  "codePath": "stage1.py",
+  "email": "gia0603yucca@gmail.com",
+  "root": "./all_checkpoints/stage1_06262112/",
+  "host": "dsw-265304-cd576ddc5-gh74w",
+  "executable": "/root/miniconda3/envs/protT3/bin/python",
+  "codePathLocal": "stage1.py",
+  "cpu_count": 64,
+  "cpu_count_logical": 64,
+  "gpu": "NVIDIA A800-SXM4-80GB",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1623302262784",
+      "used": "1290924032"
+    }
+  },
+  "memory": {
+    "total": "549755813888"
+  },
+  "cpu": {
+    "count": 64,
+    "countLogical": 64
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    }
+  ],
+  "cudaVersion": "12.1"
+}

ProtT3/all_checkpoints/stage1_06262112/wandb/run-20250626_211359-gp8ndo2v/logs/debug-internal.log

@@ -0,0 +1,73 @@
+{"time":"2025-06-26T21:13:59.919018005+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.11","symlink path":"all_checkpoints/stage1_06262112/wandb/run-20250626_211359-gp8ndo2v/logs/debug-core.log"}
+{"time":"2025-06-26T21:14:01.751242877+08:00","level":"INFO","msg":"created new stream","id":"gp8ndo2v"}
+{"time":"2025-06-26T21:14:01.751292945+08:00","level":"INFO","msg":"stream: started","id":"gp8ndo2v"}
+{"time":"2025-06-26T21:14:01.751353982+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"gp8ndo2v"}
+{"time":"2025-06-26T21:14:01.751376676+08:00","level":"INFO","msg":"handler: started","stream_id":"gp8ndo2v"}
+{"time":"2025-06-26T21:14:01.751406784+08:00","level":"INFO","msg":"sender: started","stream_id":"gp8ndo2v"}
+{"time":"2025-06-26T21:14:03.18201785+08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2025-06-26T21:39:10.805194559+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:48308->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-06-26T21:45:08.181153524+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:33626->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-26T21:50:43.748258238+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:39240->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-06-26T21:56:59.349224169+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:41628->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-26T22:01:29.173164681+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:41150->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-06-26T22:04:52.9491833+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:40328->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-26T22:05:33.372515641+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-26T22:10:24.214205918+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:39814->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-26T22:11:19.608808233+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": unexpected EOF"}
+{"time":"2025-06-26T22:15:15.541207766+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:57490->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-26T22:17:53.749178371+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:34226->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-06-26T22:20:43.734188539+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:58158->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-26T22:26:37.244674658+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-26T22:27:27.97084057+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:42482->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-06-26T22:32:12.373221258+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:42718->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-26T22:33:45.749714178+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:36352->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-06-26T22:34:27.154183486+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": unexpected EOF"}
+{"time":"2025-06-26T22:37:03.388715023+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-26T22:38:44.053145624+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:56120->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-26T22:39:26.21620593+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:37762->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-06-26T22:42:48.392517517+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-26T22:43:20.509939526+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-06-26T22:43:55.50812991+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-06-26T22:44:05.260626832+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": context deadline exceeded"}
+{"time":"2025-06-26T22:44:33.704733361+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-26T22:47:28.34118454+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:46752->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-26T22:48:22.800067638+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:35900->104.21.20.172:443: read: connection reset by peer"}
+{"time":"2025-06-26T22:49:03.396821287+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-26T22:49:35.881823651+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-06-26T22:50:10.244289946+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-06-26T22:50:48.344767175+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-06-26T22:51:21.302223032+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:40288->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-06-26T22:54:35.861164416+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:35106->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-26T22:56:18.401507947+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-26T22:56:50.726216671+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-06-26T22:57:25.218970516+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-26T22:58:04.180971507+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-06-26T22:58:09.302334148+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:50854->104.21.20.172:443: read: connection reset by peer"}
+{"time":"2025-06-26T22:58:51.546103299+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-26T23:00:01.390576029+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-26T23:02:34.06917085+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:37644->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-26T23:05:03.407063467+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-26T23:06:15.25317264+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:42244->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-06-26T23:06:33.40769986+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-06-26T23:09:18.410102964+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-26T23:11:33.411287647+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-26T23:12:46.933205687+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:55392->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-26T23:16:18.41464022+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-26T23:16:40.753618008+08:00","level":"ERROR","msg":"filestream: json decode error: net/http: request canceled (Client.Timeout or context cancellation while reading body)"}
+{"time":"2025-06-26T23:16:50.454824576+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-06-26T23:17:25.158378302+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-26T23:18:00.15504283+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": unexpected EOF"}
+{"time":"2025-06-26T23:18:03.880963877+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-26T23:18:50.836175421+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-06-26T23:20:45.653178557+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:38816->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-26T23:21:51.085821178+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:43680->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-06-26T23:24:50.901213106+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:40734->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-26T23:30:06.293188303+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:46360->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-06-26T23:36:26.709172933+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:33352->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-06-26T23:39:53.889169333+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": unexpected EOF"}
+{"time":"2025-06-26T23:42:48.781078984+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": read tcp 10.1.8.160:35500->104.21.20.172:443: read: connection reset by peer"}
+{"time":"2025-06-26T23:44:39.969525919+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:51204->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-06-26T23:52:25.685198314+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage1_06262112/gp8ndo2v/file_stream\": read tcp 10.1.8.160:41504->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-06-27T00:14:48.690187795+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": read tcp 10.1.8.160:49576->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-06-27T00:15:11.63645902+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": unexpected EOF"}
+{"time":"2025-06-27T00:18:18.739690809+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": read tcp 10.1.8.160:55462->172.67.193.61:443: read: connection reset by peer"}

ProtT3/all_checkpoints/stage1_06262112/wandb/run-20250626_211359-gp8ndo2v/logs/debug.log

@@ -0,0 +1,23 @@
+2025-06-26 21:13:59,905 INFO    MainThread:183028 [wandb_setup.py:_flush():70] Current SDK version is 0.19.11
+2025-06-26 21:13:59,905 INFO    MainThread:183028 [wandb_setup.py:_flush():70] Configure stats pid to 183028
+2025-06-26 21:13:59,905 INFO    MainThread:183028 [wandb_setup.py:_flush():70] Loading settings from /root/.config/wandb/settings
+2025-06-26 21:13:59,905 INFO    MainThread:183028 [wandb_setup.py:_flush():70] Loading settings from /nas/shared/kilab/wangyujia/ProtT3/wandb/settings
+2025-06-26 21:13:59,905 INFO    MainThread:183028 [wandb_setup.py:_flush():70] Loading settings from environment variables
+2025-06-26 21:13:59,905 INFO    MainThread:183028 [wandb_init.py:setup_run_log_directory():724] Logging user logs to ./all_checkpoints/stage1_06262112/wandb/run-20250626_211359-gp8ndo2v/logs/debug.log
+2025-06-26 21:13:59,905 INFO    MainThread:183028 [wandb_init.py:setup_run_log_directory():725] Logging internal logs to ./all_checkpoints/stage1_06262112/wandb/run-20250626_211359-gp8ndo2v/logs/debug-internal.log
+2025-06-26 21:13:59,905 INFO    MainThread:183028 [wandb_init.py:init():852] calling init triggers
+2025-06-26 21:13:59,905 INFO    MainThread:183028 [wandb_init.py:init():857] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-06-26 21:13:59,905 INFO    MainThread:183028 [wandb_init.py:init():893] starting backend
+2025-06-26 21:13:59,905 INFO    MainThread:183028 [wandb_init.py:init():897] sending inform_init request
+2025-06-26 21:13:59,906 INFO    MainThread:183028 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-06-26 21:13:59,908 INFO    MainThread:183028 [wandb_init.py:init():907] backend started and connected
+2025-06-26 21:13:59,909 INFO    MainThread:183028 [wandb_init.py:init():1005] updated telemetry
+2025-06-26 21:13:59,911 INFO    MainThread:183028 [wandb_init.py:init():1029] communicating run to backend with 90.0 second timeout
+2025-06-26 21:14:03,171 INFO    MainThread:183028 [wandb_init.py:init():1104] starting run threads in backend
+2025-06-26 21:14:03,332 INFO    MainThread:183028 [wandb_run.py:_console_start():2573] atexit reg
+2025-06-26 21:14:03,333 INFO    MainThread:183028 [wandb_run.py:_redirect():2421] redirect: wrap_raw
+2025-06-26 21:14:03,336 INFO    MainThread:183028 [wandb_run.py:_redirect():2490] Wrapping output streams.
+2025-06-26 21:14:03,336 INFO    MainThread:183028 [wandb_run.py:_redirect():2513] Redirects installed.
+2025-06-26 21:14:03,350 INFO    MainThread:183028 [wandb_init.py:init():1150] run started, returning control to user process
+2025-06-26 21:14:09,516 INFO    MainThread:183028 [wandb_run.py:_config_callback():1436] config_cb None None {'filename': 'stage1_06262112', 'seed': 42, 'mode': 'train', 'strategy': 'deepspeed', 'accelerator': 'gpu', 'devices': '0,1,2,3,4,5,6,7', 'precision': 'bf16-mixed', 'max_epochs': 30, 'check_val_every_n_epoch': 1, 'use_wandb_logger': True, 'mix_dataset': True, 'temperature': 0.1, 'save_every_n_epochs': 5, 'ptm': True, 'lm': True, 'rerank_cand_num': 128, 'plm_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m', 'plm_tune': 'freeze', 'load_4bit': False, 'pool_size': 0, 'bert_hidden_dim': 768, 'bert_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft', 'projection_dim': 256, 'cross_attention_freq': 2, 'num_query_token': 8, 'weight_decay': 0.05, 'init_lr': 0.0001, 'min_lr': 1e-05, 'warmup_lr': 1e-06, 'warmup_steps': 1000, 'lr_decay_rate': 0.9, 'scheduler': 'linear_warmup_cosine_lr', 'init_checkpoint': '', 'retrieval_eval_epoch': 10, 'num_workers': 8, 'batch_size': 160, 'match_batch_size': 64, 'root': 'data', 'text_max_len': 128, 'prot_max_len': 1024, 'prot_aug': 'None'}

ProtT3/all_checkpoints/stage2.5_mol_instruction/wandb/debug-internal.log

@@ -0,0 +1,17 @@
+{"time":"2025-07-07T20:02:29.75666986+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.11","symlink path":"all_checkpoints/stage2.5_mol_instruction/wandb/run-20250707_200229-yex1pcwt/logs/debug-core.log"}
+{"time":"2025-07-07T20:02:30.922635932+08:00","level":"INFO","msg":"created new stream","id":"yex1pcwt"}
+{"time":"2025-07-07T20:02:30.922678667+08:00","level":"INFO","msg":"stream: started","id":"yex1pcwt"}
+{"time":"2025-07-07T20:02:30.922713833+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"yex1pcwt"}
+{"time":"2025-07-07T20:02:30.922757284+08:00","level":"INFO","msg":"sender: started","stream_id":"yex1pcwt"}
+{"time":"2025-07-07T20:02:30.92278615+08:00","level":"INFO","msg":"handler: started","stream_id":"yex1pcwt"}
+{"time":"2025-07-07T20:02:32.296458789+08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2025-07-08T01:01:50.39071972+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": read tcp 10.1.4.159:39416->104.21.20.172:443: read: connection reset by peer"}
+{"time":"2025-07-08T01:07:45.887474022+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2.5_mol_instruction/yex1pcwt/file_stream\": read tcp 10.1.4.159:48838->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-08T05:20:12.207426797+08:00","level":"INFO","msg":"stream: closing","id":"yex1pcwt"}
+{"time":"2025-07-08T05:20:12.207468139+08:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2025-07-08T05:20:12.208684636+08:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2025-07-08T05:20:13.938647534+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-07-08T05:20:14.977621841+08:00","level":"INFO","msg":"handler: closed","stream_id":"yex1pcwt"}
+{"time":"2025-07-08T05:20:14.977653692+08:00","level":"INFO","msg":"sender: closed","stream_id":"yex1pcwt"}
+{"time":"2025-07-08T05:20:14.977651902+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"yex1pcwt"}
+{"time":"2025-07-08T05:20:14.982274952+08:00","level":"INFO","msg":"stream: closed","id":"yex1pcwt"}

ProtT3/all_checkpoints/stage2.5_mol_instruction/wandb/debug.log

@@ -0,0 +1,24 @@
+2025-07-07 20:02:29,743 INFO    MainThread:129761 [wandb_setup.py:_flush():70] Current SDK version is 0.19.11
+2025-07-07 20:02:29,743 INFO    MainThread:129761 [wandb_setup.py:_flush():70] Configure stats pid to 129761
+2025-07-07 20:02:29,743 INFO    MainThread:129761 [wandb_setup.py:_flush():70] Loading settings from /root/.config/wandb/settings
+2025-07-07 20:02:29,743 INFO    MainThread:129761 [wandb_setup.py:_flush():70] Loading settings from /nas/shared/kilab/wangyujia/ProtT3/wandb/settings
+2025-07-07 20:02:29,743 INFO    MainThread:129761 [wandb_setup.py:_flush():70] Loading settings from environment variables
+2025-07-07 20:02:29,743 INFO    MainThread:129761 [wandb_init.py:setup_run_log_directory():724] Logging user logs to ./all_checkpoints/stage2.5_mol_instruction/wandb/run-20250707_200229-yex1pcwt/logs/debug.log
+2025-07-07 20:02:29,743 INFO    MainThread:129761 [wandb_init.py:setup_run_log_directory():725] Logging internal logs to ./all_checkpoints/stage2.5_mol_instruction/wandb/run-20250707_200229-yex1pcwt/logs/debug-internal.log
+2025-07-07 20:02:29,743 INFO    MainThread:129761 [wandb_init.py:init():852] calling init triggers
+2025-07-07 20:02:29,743 INFO    MainThread:129761 [wandb_init.py:init():857] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-07-07 20:02:29,743 INFO    MainThread:129761 [wandb_init.py:init():893] starting backend
+2025-07-07 20:02:29,743 INFO    MainThread:129761 [wandb_init.py:init():897] sending inform_init request
+2025-07-07 20:02:29,745 INFO    MainThread:129761 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-07-07 20:02:29,748 INFO    MainThread:129761 [wandb_init.py:init():907] backend started and connected
+2025-07-07 20:02:29,754 INFO    MainThread:129761 [wandb_init.py:init():1005] updated telemetry
+2025-07-07 20:02:29,762 INFO    MainThread:129761 [wandb_init.py:init():1029] communicating run to backend with 90.0 second timeout
+2025-07-07 20:02:32,258 INFO    MainThread:129761 [wandb_init.py:init():1104] starting run threads in backend
+2025-07-07 20:02:32,427 INFO    MainThread:129761 [wandb_run.py:_console_start():2573] atexit reg
+2025-07-07 20:02:32,427 INFO    MainThread:129761 [wandb_run.py:_redirect():2421] redirect: wrap_raw
+2025-07-07 20:02:32,457 INFO    MainThread:129761 [wandb_run.py:_redirect():2490] Wrapping output streams.
+2025-07-07 20:02:32,462 INFO    MainThread:129761 [wandb_run.py:_redirect():2513] Redirects installed.
+2025-07-07 20:02:32,463 INFO    MainThread:129761 [wandb_init.py:init():1150] run started, returning control to user process
+2025-07-07 20:02:40,689 INFO    MainThread:129761 [wandb_run.py:_config_callback():1436] config_cb None None {'filename': 'stage2.5_mol_instruction', 'seed': 42, 'mode': 'train', 'strategy': 'deepspeed', 'accelerator': 'gpu', 'devices': '0,1,2,3,4,5,6,7', 'precision': 'bf16-mixed', 'max_epochs': 10, 'accumulate_grad_batches': 1, 'check_val_every_n_epoch': 1, 'enable_flash': False, 'use_wandb_logger': True, 'mix_dataset': False, 'save_every_n_epochs': 1, 'bert_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft', 'cross_attention_freq': 2, 'num_query_token': 8, 'llm_name': '/oss/wangyujia/BIO/pretrain_output/qwen2.5-7b-instruct-bio/bio_all/save1epoch/checkpoint-1300', 'num_beams': 5, 'do_sample': False, 'max_inference_len': 128, 'min_inference_len': 1, 'llm_tune': 'mid_lora', 'peft_config': '', 'peft_dir': '', 'plm_model': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m', 'plm_tune': 'freeze', 'lora_r': 8, 'lora_alpha': 16, 'lora_dropout': 0.1, 'enbale_gradient_checkpointing': False, 'weight_decay': 0.05, 'init_lr': 0.0001, 'min_lr': 1e-05, 'warmup_lr': 1e-06, 'warmup_steps': 1000, 'lr_decay_rate': 0.9, 'scheduler': 'linear_warmup_cosine_lr', 'stage1_path': '/nas/shared/kilab/wangyujia/ProtT3/all_checkpoints/stage1_06290009_deepspeed/epoch=19.ckpt/converted.ckpt', 'stage2_path': '', 'init_checkpoint': '', 'caption_eval_epoch': 10, 'num_workers': 8, 'batch_size': 2, 'inference_batch_size': 4, 'root': 'data', 'text_max_len': 1024, 'q_max_len': 29, 'a_max_len': 36, 'prot_max_len': 1024, 'prompt': 'The protein has the following properties: ', 'filter_side_qa': False}
+2025-07-08 05:20:12,205 INFO    MsgRouterThr:129761 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles.

ProtT3/all_checkpoints/stage2.5_mol_instruction/wandb/run-20250707_184942-2bo0nfvt/files/output.log

@@ -0,0 +1,16 @@
+/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:654: Checkpoint directory /nas/shared/kilab/wangyujia/ProtT3/all_checkpoints/stage2.5_mol_instruction exists and is not empty.
+Enabling DeepSpeed BF16. Model parameters and inputs will be cast to `bfloat16`.
+LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]
+
+  | Name  | Type     | Params | Mode
+-------------------------------------------
+0 | blip2 | Blip2OPT | 7.9 B  | train
+-------------------------------------------
+104 M     Trainable params
+7.8 B     Non-trainable params
+7.9 B     Total params
+31,459.025Total estimated model params size (MB)
+174       Modules in train mode
+1203      Modules in eval mode
+Epoch 0:   0%|                                                         | 0/410 [00:00<?, ?it/s]
+[rank: 1] Child process with PID 111788 terminated with code 1. Forcefully terminating all other processes to avoid zombies 🧟

ProtT3/all_checkpoints/stage2.5_mol_instruction/wandb/run-20250707_184942-2bo0nfvt/files/requirements.txt

@@ -0,0 +1,225 @@
+pydantic_core==2.33.2
+psutil==7.0.0
+nvidia-cuda-nvrtc-cu12==12.4.127
+mpmath==1.3.0
+tzdata==2025.2
+contexttimer==0.3.3
+cycler==0.12.1
+python-magic==0.4.27
+pexpect==4.9.0
+sympy==1.13.1
+wrapt==1.17.2
+marisa-trie==1.2.1
+langcodes==3.5.0
+nvidia-nvtx-cu12==12.4.127
+ipython==8.36.0
+opencv-python-headless==4.5.5.64
+MarkupSafe==3.0.2
+jsonschema-specifications==2025.4.1
+wasabi==1.1.3
+blinker==1.9.0
+cfgv==3.4.0
+numpy==2.2.6
+idna==3.10
+nvidia-cufile-cu12==1.11.1.6
+ninja==1.11.1.4
+nvidia-nccl-cu12==2.21.5
+networkx==3.4.2
+certifi==2025.4.26
+deepspeed==0.16.10+b666844f
+pure_eval==0.2.3
+packaging==24.2
+nltk==3.9.1
+contourpy==1.3.2
+pre_commit==4.2.0
+nodeenv==1.9.1
+setuptools==78.1.1
+annotated-types==0.7.0
+multidict==6.4.4
+requests==2.32.3
+tornado==6.5.1
+triton==3.2.0
+pillow==11.2.1
+decord==0.6.0
+shellingham==1.5.4
+streamlit==1.45.1
+pydeck==0.9.1
+confection==0.1.5
+exceptiongroup==1.3.0
+prompt_toolkit==3.0.51
+text-unidecode==1.3
+nvidia-cufft-cu12==11.2.1.3
+antlr4-python3-runtime==4.9.3
+fairscale==0.4.4
+rouge_score==0.1.2
+nvidia-cudnn-cu12==9.1.0.70
+tqdm==4.67.1
+rich==14.0.0
+frozenlist==1.6.0
+webencodings==0.5.1
+altair==5.5.0
+opendatasets==0.1.22
+nvidia-curand-cu12==10.3.5.147
+protobuf==6.31.0
+asttokens==3.0.0
+wheel==0.45.1
+hf-xet==1.1.2
+weasel==0.4.1
+aiosignal==1.3.2
+absl-py==2.2.2
+thinc==8.3.6
+torchvision==0.21.0
+pandas==2.2.3
+fonttools==4.58.0
+bleach==6.2.0
+typing-inspection==0.4.1
+ftfy==6.3.1
+typing_extensions==4.13.2
+nvidia-ml-py==12.575.51
+python-slugify==8.0.4
+lightning-utilities==0.14.3
+py-cpuinfo==9.0.0
+smmap==5.0.2
+regex==2024.11.6
+scikit-image==0.25.2
+iopath==0.1.10
+spacy-legacy==3.0.12
+hjson==3.1.0
+executing==2.2.0
+kiwisolver==1.4.8
+scipy==1.15.3
+aiohappyeyeballs==2.6.1
+toml==0.10.2
+jedi==0.19.2
+GitPython==3.1.44
+ptyprocess==0.7.0
+kaggle==1.7.4.5
+braceexpand==0.1.7
+wcwidth==0.2.13
+nvidia-cuda-runtime-cu12==12.4.127
+pytorch-lightning==2.5.1.post0
+Jinja2==3.1.6
+urllib3==2.4.0
+watchdog==6.0.0
+filelock==3.18.0
+propcache==0.3.1
+torch==2.6.0
+nvidia-cusparse-cu12==12.3.1.170
+cymem==2.0.11
+nvidia-cusolver-cu12==11.6.1.9
+murmurhash==1.0.13
+catalogue==2.0.10
+yarl==1.20.0
+charset-normalizer==3.4.2
+gitdb==4.0.12
+matplotlib==3.10.3
+portalocker==3.1.1
+platformdirs==4.3.8
+async-timeout==5.0.1
+parso==0.8.4
+markdown-it-py==3.0.0
+omegaconf==2.3.0
+cloudpathlib==0.21.1
+nvidia-cusparselt-cu12==0.6.2
+spacy-loggers==1.0.5
+srsly==2.5.1
+identify==2.6.12
+rpds-py==0.25.1
+spacy==3.8.7
+matplotlib-inline==0.1.7
+smart-open==7.1.0
+pydantic==2.11.5
+mdurl==0.1.2
+virtualenv==20.31.2
+pytz==2025.2
+pycocotools==2.0.8
+six==1.17.0
+decorator==5.2.1
+referencing==0.36.2
+sentencepiece==0.2.0
+PyYAML==6.0.2
+pycocoevalcap==1.2
+imageio==2.37.0
+distlib==0.3.9
+pyarrow==20.0.0
+tenacity==9.1.2
+language_data==1.3.0
+nvidia-cuda-cupti-cu12==12.4.127
+blis==1.3.0
+Pygments==2.19.1
+tifffile==2025.5.10
+pyparsing==3.2.3
+cachetools==5.5.2
+safetensors==0.5.3
+attrs==25.3.0
+webdataset==0.2.111
+plotly==6.1.1
+nvidia-cublas-cu12==12.4.5.8
+timm==0.4.12
+torchmetrics==1.7.1
+nvidia-nvjitlink-cu12==12.4.127
+stack-data==0.6.3
+python-dateutil==2.9.0.post0
+lazy_loader==0.4
+traitlets==5.14.3
+einops==0.8.1
+salesforce-lavis==1.0.2
+joblib==1.5.1
+msgpack==1.1.0
+tokenizers==0.21.1
+sentry-sdk==2.29.1
+oss2==2.15.0
+setproctitle==1.3.6
+pip==25.1.1
+cffi==1.17.1
+transformers==4.52.3
+narwhals==1.41.0
+aliyun-python-sdk-core==2.16.0
+jsonschema==4.24.0
+flash-attn==2.7.1.post1
+preshed==3.0.10
+multiprocess==0.70.16
+cryptography==45.0.3
+aliyun-python-sdk-kms==2.16.5
+scikit-learn==1.6.1
+huggingface-hub==0.32.1
+crcmod==1.7
+typer==0.16.0
+web.py==0.62
+docker-pycreds==0.4.0
+xxhash==3.5.0
+bigmodelvis==0.0.1
+datasets==3.6.0
+more-itertools==10.7.0
+yacs==0.1.8
+jmespath==0.10.0
+aiohttp==3.12.2
+opencv-python==4.11.0.86
+pycparser==2.22
+threadpoolctl==3.6.0
+jaraco.functools==4.1.0
+click==8.2.1
+wandb==0.19.11
+opendelta==0.3.2
+pycryptodome==3.23.0
+pathlib==1.0.1
+dill==0.3.8
+fsspec==2025.3.0
+delta-center-client==0.0.4
+cheroot==10.0.1
+typing_extensions==4.12.2
+platformdirs==4.2.2
+jaraco.text==3.12.1
+packaging==24.2
+inflect==7.3.1
+jaraco.context==5.3.0
+wheel==0.45.1
+typeguard==4.3.0
+more-itertools==10.3.0
+tomli==2.0.1
+importlib_metadata==8.0.0
+backports.tarfile==1.2.0
+zipp==3.19.2
+jaraco.collections==5.1.0
+autocommand==2.2.2
+jaraco.functools==4.0.1

ProtT3/all_checkpoints/stage2.5_mol_instruction/wandb/run-20250707_184942-2bo0nfvt/files/wandb-metadata.json

@@ -0,0 +1,103 @@
+{
+  "os": "Linux-5.10.134-008.16.kangaroo.al8.x86_64-x86_64-with-glibc2.35",
+  "python": "CPython 3.10.0",
+  "startedAt": "2025-07-07T10:49:42.891959Z",
+  "args": [
+    "--devices",
+    "0,1,2,3,4,5,6,7",
+    "--mode",
+    "train",
+    "--filename",
+    "stage2.5_mol_instruction",
+    "--num_query_token",
+    "8",
+    "--save_every_n_epochs",
+    "1",
+    "--max_epochs",
+    "10",
+    "--batch_size",
+    "32",
+    "--precision",
+    "bf16-mixed",
+    "--num_workers",
+    "8",
+    "--plm_model",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m",
+    "--bert_name",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft",
+    "--llm_name",
+    "/oss/wangyujia/BIO/pretrain_output/qwen2.5-7b-instruct-bio/bio_all/save1epoch/checkpoint-1300",
+    "--llm_tune",
+    "mid_lora",
+    "--stage1_path",
+    "/nas/shared/kilab/wangyujia/ProtT3/all_checkpoints/stage1_06290009_deepspeed/epoch=19.ckpt/converted.ckpt",
+    "--use_wandb_logger"
+  ],
+  "program": "/nas/shared/kilab/wangyujia/ProtT3/stage2.py",
+  "codePath": "stage2.py",
+  "email": "gia0603yucca@gmail.com",
+  "root": "./all_checkpoints/stage2.5_mol_instruction/",
+  "host": "dsw-265304-7f6db6b4bb-g4b9r",
+  "executable": "/root/miniconda3/envs/protT3/bin/python",
+  "codePathLocal": "stage2.py",
+  "cpu_count": 64,
+  "cpu_count_logical": 64,
+  "gpu": "NVIDIA A800-SXM4-80GB",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1623302262784",
+      "used": "1260912640"
+    }
+  },
+  "memory": {
+    "total": "549755813888"
+  },
+  "cpu": {
+    "count": 64,
+    "countLogical": 64
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    }
+  ],
+  "cudaVersion": "12.1"
+}

ProtT3/all_checkpoints/stage2.5_mol_instruction/wandb/run-20250707_184942-2bo0nfvt/logs/debug-internal.log

@@ -0,0 +1,7 @@
+{"time":"2025-07-07T18:49:42.893783985+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.11","symlink path":"all_checkpoints/stage2.5_mol_instruction/wandb/run-20250707_184942-2bo0nfvt/logs/debug-core.log"}
+{"time":"2025-07-07T18:49:43.936695317+08:00","level":"INFO","msg":"created new stream","id":"2bo0nfvt"}
+{"time":"2025-07-07T18:49:43.936731645+08:00","level":"INFO","msg":"stream: started","id":"2bo0nfvt"}
+{"time":"2025-07-07T18:49:43.936758154+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"2bo0nfvt"}
+{"time":"2025-07-07T18:49:43.936774344+08:00","level":"INFO","msg":"sender: started","stream_id":"2bo0nfvt"}
+{"time":"2025-07-07T18:49:43.936811419+08:00","level":"INFO","msg":"handler: started","stream_id":"2bo0nfvt"}
+{"time":"2025-07-07T18:49:45.07418554+08:00","level":"INFO","msg":"Starting system monitor"}

ProtT3/all_checkpoints/stage2.5_mol_instruction/wandb/run-20250707_184942-2bo0nfvt/logs/debug.log

@@ -0,0 +1,23 @@
+2025-07-07 18:49:42,884 INFO    MainThread:111335 [wandb_setup.py:_flush():70] Current SDK version is 0.19.11
+2025-07-07 18:49:42,884 INFO    MainThread:111335 [wandb_setup.py:_flush():70] Configure stats pid to 111335
+2025-07-07 18:49:42,884 INFO    MainThread:111335 [wandb_setup.py:_flush():70] Loading settings from /root/.config/wandb/settings
+2025-07-07 18:49:42,884 INFO    MainThread:111335 [wandb_setup.py:_flush():70] Loading settings from /nas/shared/kilab/wangyujia/ProtT3/wandb/settings
+2025-07-07 18:49:42,884 INFO    MainThread:111335 [wandb_setup.py:_flush():70] Loading settings from environment variables
+2025-07-07 18:49:42,884 INFO    MainThread:111335 [wandb_init.py:setup_run_log_directory():724] Logging user logs to ./all_checkpoints/stage2.5_mol_instruction/wandb/run-20250707_184942-2bo0nfvt/logs/debug.log
+2025-07-07 18:49:42,884 INFO    MainThread:111335 [wandb_init.py:setup_run_log_directory():725] Logging internal logs to ./all_checkpoints/stage2.5_mol_instruction/wandb/run-20250707_184942-2bo0nfvt/logs/debug-internal.log
+2025-07-07 18:49:42,884 INFO    MainThread:111335 [wandb_init.py:init():852] calling init triggers
+2025-07-07 18:49:42,884 INFO    MainThread:111335 [wandb_init.py:init():857] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-07-07 18:49:42,884 INFO    MainThread:111335 [wandb_init.py:init():893] starting backend
+2025-07-07 18:49:42,884 INFO    MainThread:111335 [wandb_init.py:init():897] sending inform_init request
+2025-07-07 18:49:42,886 INFO    MainThread:111335 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-07-07 18:49:42,888 INFO    MainThread:111335 [wandb_init.py:init():907] backend started and connected
+2025-07-07 18:49:42,893 INFO    MainThread:111335 [wandb_init.py:init():1005] updated telemetry
+2025-07-07 18:49:42,896 INFO    MainThread:111335 [wandb_init.py:init():1029] communicating run to backend with 90.0 second timeout
+2025-07-07 18:49:45,025 INFO    MainThread:111335 [wandb_init.py:init():1104] starting run threads in backend
+2025-07-07 18:49:45,210 INFO    MainThread:111335 [wandb_run.py:_console_start():2573] atexit reg
+2025-07-07 18:49:45,210 INFO    MainThread:111335 [wandb_run.py:_redirect():2421] redirect: wrap_raw
+2025-07-07 18:49:45,214 INFO    MainThread:111335 [wandb_run.py:_redirect():2490] Wrapping output streams.
+2025-07-07 18:49:45,214 INFO    MainThread:111335 [wandb_run.py:_redirect():2513] Redirects installed.
+2025-07-07 18:49:45,216 INFO    MainThread:111335 [wandb_init.py:init():1150] run started, returning control to user process
+2025-07-07 18:49:53,747 INFO    MainThread:111335 [wandb_run.py:_config_callback():1436] config_cb None None {'filename': 'stage2.5_mol_instruction', 'seed': 42, 'mode': 'train', 'strategy': 'deepspeed', 'accelerator': 'gpu', 'devices': '0,1,2,3,4,5,6,7', 'precision': 'bf16-mixed', 'max_epochs': 10, 'accumulate_grad_batches': 1, 'check_val_every_n_epoch': 1, 'enable_flash': False, 'use_wandb_logger': True, 'mix_dataset': False, 'save_every_n_epochs': 1, 'bert_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft', 'cross_attention_freq': 2, 'num_query_token': 8, 'llm_name': '/oss/wangyujia/BIO/pretrain_output/qwen2.5-7b-instruct-bio/bio_all/save1epoch/checkpoint-1300', 'num_beams': 5, 'do_sample': False, 'max_inference_len': 128, 'min_inference_len': 1, 'llm_tune': 'mid_lora', 'peft_config': '', 'peft_dir': '', 'plm_model': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m', 'plm_tune': 'freeze', 'lora_r': 8, 'lora_alpha': 16, 'lora_dropout': 0.1, 'enbale_gradient_checkpointing': False, 'weight_decay': 0.05, 'init_lr': 0.0001, 'min_lr': 1e-05, 'warmup_lr': 1e-06, 'warmup_steps': 1000, 'lr_decay_rate': 0.9, 'scheduler': 'linear_warmup_cosine_lr', 'stage1_path': '/nas/shared/kilab/wangyujia/ProtT3/all_checkpoints/stage1_06290009_deepspeed/epoch=19.ckpt/converted.ckpt', 'stage2_path': '', 'init_checkpoint': '', 'caption_eval_epoch': 10, 'num_workers': 8, 'batch_size': 32, 'inference_batch_size': 4, 'root': 'data', 'text_max_len': 1024, 'q_max_len': 29, 'a_max_len': 36, 'prot_max_len': 1024, 'prompt': 'The protein has the following properties: ', 'filter_side_qa': False}

ProtT3/all_checkpoints/stage2.5_mol_instruction/wandb/run-20250707_185239-ftp1v3gy/files/output.log

@@ -0,0 +1,4 @@
+/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:654: Checkpoint directory /nas/shared/kilab/wangyujia/ProtT3/all_checkpoints/stage2.5_mol_instruction exists and is not empty.
+Enabling DeepSpeed BF16. Model parameters and inputs will be cast to `bfloat16`.
+LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]
+[rank: 5] Child process with PID 116132 terminated with code 1. Forcefully terminating all other processes to avoid zombies 🧟

ProtT3/all_checkpoints/stage2.5_mol_instruction/wandb/run-20250707_185239-ftp1v3gy/files/requirements.txt

@@ -0,0 +1,225 @@
+pydantic_core==2.33.2
+psutil==7.0.0
+nvidia-cuda-nvrtc-cu12==12.4.127
+mpmath==1.3.0
+tzdata==2025.2
+contexttimer==0.3.3
+cycler==0.12.1
+python-magic==0.4.27
+pexpect==4.9.0
+sympy==1.13.1
+wrapt==1.17.2
+marisa-trie==1.2.1
+langcodes==3.5.0
+nvidia-nvtx-cu12==12.4.127
+ipython==8.36.0
+opencv-python-headless==4.5.5.64
+MarkupSafe==3.0.2
+jsonschema-specifications==2025.4.1
+wasabi==1.1.3
+blinker==1.9.0
+cfgv==3.4.0
+numpy==2.2.6
+idna==3.10
+nvidia-cufile-cu12==1.11.1.6
+ninja==1.11.1.4
+nvidia-nccl-cu12==2.21.5
+networkx==3.4.2
+certifi==2025.4.26
+deepspeed==0.16.10+b666844f
+pure_eval==0.2.3
+packaging==24.2
+nltk==3.9.1
+contourpy==1.3.2
+pre_commit==4.2.0
+nodeenv==1.9.1
+setuptools==78.1.1
+annotated-types==0.7.0
+multidict==6.4.4
+requests==2.32.3
+tornado==6.5.1
+triton==3.2.0
+pillow==11.2.1
+decord==0.6.0
+shellingham==1.5.4
+streamlit==1.45.1
+pydeck==0.9.1
+confection==0.1.5
+exceptiongroup==1.3.0
+prompt_toolkit==3.0.51
+text-unidecode==1.3
+nvidia-cufft-cu12==11.2.1.3
+antlr4-python3-runtime==4.9.3
+fairscale==0.4.4
+rouge_score==0.1.2
+nvidia-cudnn-cu12==9.1.0.70
+tqdm==4.67.1
+rich==14.0.0
+frozenlist==1.6.0
+webencodings==0.5.1
+altair==5.5.0
+opendatasets==0.1.22
+nvidia-curand-cu12==10.3.5.147
+protobuf==6.31.0
+asttokens==3.0.0
+wheel==0.45.1
+hf-xet==1.1.2
+weasel==0.4.1
+aiosignal==1.3.2
+absl-py==2.2.2
+thinc==8.3.6
+torchvision==0.21.0
+pandas==2.2.3
+fonttools==4.58.0
+bleach==6.2.0
+typing-inspection==0.4.1
+ftfy==6.3.1
+typing_extensions==4.13.2
+nvidia-ml-py==12.575.51
+python-slugify==8.0.4
+lightning-utilities==0.14.3
+py-cpuinfo==9.0.0
+smmap==5.0.2
+regex==2024.11.6
+scikit-image==0.25.2
+iopath==0.1.10
+spacy-legacy==3.0.12
+hjson==3.1.0
+executing==2.2.0
+kiwisolver==1.4.8
+scipy==1.15.3
+aiohappyeyeballs==2.6.1
+toml==0.10.2
+jedi==0.19.2
+GitPython==3.1.44
+ptyprocess==0.7.0
+kaggle==1.7.4.5
+braceexpand==0.1.7
+wcwidth==0.2.13
+nvidia-cuda-runtime-cu12==12.4.127
+pytorch-lightning==2.5.1.post0
+Jinja2==3.1.6
+urllib3==2.4.0
+watchdog==6.0.0
+filelock==3.18.0
+propcache==0.3.1
+torch==2.6.0
+nvidia-cusparse-cu12==12.3.1.170
+cymem==2.0.11
+nvidia-cusolver-cu12==11.6.1.9
+murmurhash==1.0.13
+catalogue==2.0.10
+yarl==1.20.0
+charset-normalizer==3.4.2
+gitdb==4.0.12
+matplotlib==3.10.3
+portalocker==3.1.1
+platformdirs==4.3.8
+async-timeout==5.0.1
+parso==0.8.4
+markdown-it-py==3.0.0
+omegaconf==2.3.0
+cloudpathlib==0.21.1
+nvidia-cusparselt-cu12==0.6.2
+spacy-loggers==1.0.5
+srsly==2.5.1
+identify==2.6.12
+rpds-py==0.25.1
+spacy==3.8.7
+matplotlib-inline==0.1.7
+smart-open==7.1.0
+pydantic==2.11.5
+mdurl==0.1.2
+virtualenv==20.31.2
+pytz==2025.2
+pycocotools==2.0.8
+six==1.17.0
+decorator==5.2.1
+referencing==0.36.2
+sentencepiece==0.2.0
+PyYAML==6.0.2
+pycocoevalcap==1.2
+imageio==2.37.0
+distlib==0.3.9
+pyarrow==20.0.0
+tenacity==9.1.2
+language_data==1.3.0
+nvidia-cuda-cupti-cu12==12.4.127
+blis==1.3.0
+Pygments==2.19.1
+tifffile==2025.5.10
+pyparsing==3.2.3
+cachetools==5.5.2
+safetensors==0.5.3
+attrs==25.3.0
+webdataset==0.2.111
+plotly==6.1.1
+nvidia-cublas-cu12==12.4.5.8
+timm==0.4.12
+torchmetrics==1.7.1
+nvidia-nvjitlink-cu12==12.4.127
+stack-data==0.6.3
+python-dateutil==2.9.0.post0
+lazy_loader==0.4
+traitlets==5.14.3
+einops==0.8.1
+salesforce-lavis==1.0.2
+joblib==1.5.1
+msgpack==1.1.0
+tokenizers==0.21.1
+sentry-sdk==2.29.1
+oss2==2.15.0
+setproctitle==1.3.6
+pip==25.1.1
+cffi==1.17.1
+transformers==4.52.3
+narwhals==1.41.0
+aliyun-python-sdk-core==2.16.0
+jsonschema==4.24.0
+flash-attn==2.7.1.post1
+preshed==3.0.10
+multiprocess==0.70.16
+cryptography==45.0.3
+aliyun-python-sdk-kms==2.16.5
+scikit-learn==1.6.1
+huggingface-hub==0.32.1
+crcmod==1.7
+typer==0.16.0
+web.py==0.62
+docker-pycreds==0.4.0
+xxhash==3.5.0
+bigmodelvis==0.0.1
+datasets==3.6.0
+more-itertools==10.7.0
+yacs==0.1.8
+jmespath==0.10.0
+aiohttp==3.12.2
+opencv-python==4.11.0.86
+pycparser==2.22
+threadpoolctl==3.6.0
+jaraco.functools==4.1.0
+click==8.2.1
+wandb==0.19.11
+opendelta==0.3.2
+pycryptodome==3.23.0
+pathlib==1.0.1
+dill==0.3.8
+fsspec==2025.3.0
+delta-center-client==0.0.4
+cheroot==10.0.1
+typing_extensions==4.12.2
+platformdirs==4.2.2
+jaraco.text==3.12.1
+packaging==24.2
+inflect==7.3.1
+jaraco.context==5.3.0
+wheel==0.45.1
+typeguard==4.3.0
+more-itertools==10.3.0
+tomli==2.0.1
+importlib_metadata==8.0.0
+backports.tarfile==1.2.0
+zipp==3.19.2
+jaraco.collections==5.1.0
+autocommand==2.2.2
+jaraco.functools==4.0.1

ProtT3/all_checkpoints/stage2.5_mol_instruction/wandb/run-20250707_185239-ftp1v3gy/files/wandb-metadata.json

@@ -0,0 +1,103 @@
+{
+  "os": "Linux-5.10.134-008.16.kangaroo.al8.x86_64-x86_64-with-glibc2.35",
+  "python": "CPython 3.10.0",
+  "startedAt": "2025-07-07T10:52:39.364663Z",
+  "args": [
+    "--devices",
+    "0,1,2,3,4,5,6,7",
+    "--mode",
+    "train",
+    "--filename",
+    "stage2.5_mol_instruction",
+    "--num_query_token",
+    "8",
+    "--save_every_n_epochs",
+    "1",
+    "--max_epochs",
+    "10",
+    "--batch_size",
+    "2",
+    "--precision",
+    "bf16-mixed",
+    "--num_workers",
+    "8",
+    "--plm_model",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m",
+    "--bert_name",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft",
+    "--llm_name",
+    "/oss/wangyujia/BIO/pretrain_output/qwen2.5-7b-instruct-bio/bio_all/save1epoch/checkpoint-1300",
+    "--llm_tune",
+    "mid_lora",
+    "--stage1_path",
+    "/nas/shared/kilab/wangyujia/ProtT3/all_checkpoints/stage1_06290009_deepspeed/epoch=19.ckpt/converted.ckpt",
+    "--use_wandb_logger"
+  ],
+  "program": "/nas/shared/kilab/wangyujia/ProtT3/stage2.py",
+  "codePath": "stage2.py",
+  "email": "gia0603yucca@gmail.com",
+  "root": "./all_checkpoints/stage2.5_mol_instruction/",
+  "host": "dsw-265304-7f6db6b4bb-g4b9r",
+  "executable": "/root/miniconda3/envs/protT3/bin/python",
+  "codePathLocal": "stage2.py",
+  "cpu_count": 64,
+  "cpu_count_logical": 64,
+  "gpu": "NVIDIA A800-SXM4-80GB",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1623302262784",
+      "used": "1260916736"
+    }
+  },
+  "memory": {
+    "total": "549755813888"
+  },
+  "cpu": {
+    "count": 64,
+    "countLogical": 64
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    }
+  ],
+  "cudaVersion": "12.1"
+}

ProtT3/all_checkpoints/stage2.5_mol_instruction/wandb/run-20250707_185239-ftp1v3gy/logs/debug-internal.log

@@ -0,0 +1,7 @@
+{"time":"2025-07-07T18:52:39.366061072+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.11","symlink path":"all_checkpoints/stage2.5_mol_instruction/wandb/run-20250707_185239-ftp1v3gy/logs/debug-core.log"}
+{"time":"2025-07-07T18:52:40.447272944+08:00","level":"INFO","msg":"created new stream","id":"ftp1v3gy"}
+{"time":"2025-07-07T18:52:40.447330666+08:00","level":"INFO","msg":"stream: started","id":"ftp1v3gy"}
+{"time":"2025-07-07T18:52:40.447402585+08:00","level":"INFO","msg":"handler: started","stream_id":"ftp1v3gy"}
+{"time":"2025-07-07T18:52:40.447405198+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"ftp1v3gy"}
+{"time":"2025-07-07T18:52:40.447445082+08:00","level":"INFO","msg":"sender: started","stream_id":"ftp1v3gy"}
+{"time":"2025-07-07T18:52:41.658224222+08:00","level":"INFO","msg":"Starting system monitor"}

ProtT3/all_checkpoints/stage2.5_mol_instruction/wandb/run-20250707_185239-ftp1v3gy/logs/debug.log

@@ -0,0 +1,22 @@
+2025-07-07 18:52:39,355 INFO    MainThread:115698 [wandb_setup.py:_flush():70] Current SDK version is 0.19.11
+2025-07-07 18:52:39,356 INFO    MainThread:115698 [wandb_setup.py:_flush():70] Configure stats pid to 115698
+2025-07-07 18:52:39,356 INFO    MainThread:115698 [wandb_setup.py:_flush():70] Loading settings from /root/.config/wandb/settings
+2025-07-07 18:52:39,356 INFO    MainThread:115698 [wandb_setup.py:_flush():70] Loading settings from /nas/shared/kilab/wangyujia/ProtT3/wandb/settings
+2025-07-07 18:52:39,356 INFO    MainThread:115698 [wandb_setup.py:_flush():70] Loading settings from environment variables
+2025-07-07 18:52:39,356 INFO    MainThread:115698 [wandb_init.py:setup_run_log_directory():724] Logging user logs to ./all_checkpoints/stage2.5_mol_instruction/wandb/run-20250707_185239-ftp1v3gy/logs/debug.log
+2025-07-07 18:52:39,356 INFO    MainThread:115698 [wandb_init.py:setup_run_log_directory():725] Logging internal logs to ./all_checkpoints/stage2.5_mol_instruction/wandb/run-20250707_185239-ftp1v3gy/logs/debug-internal.log
+2025-07-07 18:52:39,356 INFO    MainThread:115698 [wandb_init.py:init():852] calling init triggers
+2025-07-07 18:52:39,356 INFO    MainThread:115698 [wandb_init.py:init():857] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-07-07 18:52:39,356 INFO    MainThread:115698 [wandb_init.py:init():893] starting backend
+2025-07-07 18:52:39,356 INFO    MainThread:115698 [wandb_init.py:init():897] sending inform_init request
+2025-07-07 18:52:39,357 INFO    MainThread:115698 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-07-07 18:52:39,361 INFO    MainThread:115698 [wandb_init.py:init():907] backend started and connected
+2025-07-07 18:52:39,365 INFO    MainThread:115698 [wandb_init.py:init():1005] updated telemetry
+2025-07-07 18:52:39,368 INFO    MainThread:115698 [wandb_init.py:init():1029] communicating run to backend with 90.0 second timeout
+2025-07-07 18:52:41,647 INFO    MainThread:115698 [wandb_init.py:init():1104] starting run threads in backend
+2025-07-07 18:52:41,785 INFO    MainThread:115698 [wandb_run.py:_console_start():2573] atexit reg
+2025-07-07 18:52:41,785 INFO    MainThread:115698 [wandb_run.py:_redirect():2421] redirect: wrap_raw
+2025-07-07 18:52:41,788 INFO    MainThread:115698 [wandb_run.py:_redirect():2490] Wrapping output streams.
+2025-07-07 18:52:41,788 INFO    MainThread:115698 [wandb_run.py:_redirect():2513] Redirects installed.
+2025-07-07 18:52:41,789 INFO    MainThread:115698 [wandb_init.py:init():1150] run started, returning control to user process

ProtT3/all_checkpoints/stage2.5_mol_instruction/wandb/run-20250707_185440-p1815hm9/files/output.log

@@ -0,0 +1,4 @@
+/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:654: Checkpoint directory /nas/shared/kilab/wangyujia/ProtT3/all_checkpoints/stage2.5_mol_instruction exists and is not empty.
+Enabling DeepSpeed BF16. Model parameters and inputs will be cast to `bfloat16`.
+LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]
+[rank: 5] Child process with PID 118872 terminated with code 1. Forcefully terminating all other processes to avoid zombies 🧟

ProtT3/all_checkpoints/stage2.5_mol_instruction/wandb/run-20250707_185440-p1815hm9/files/requirements.txt

@@ -0,0 +1,225 @@
+pydantic_core==2.33.2
+psutil==7.0.0
+nvidia-cuda-nvrtc-cu12==12.4.127
+mpmath==1.3.0
+tzdata==2025.2
+contexttimer==0.3.3
+cycler==0.12.1
+python-magic==0.4.27
+pexpect==4.9.0
+sympy==1.13.1
+wrapt==1.17.2
+marisa-trie==1.2.1
+langcodes==3.5.0
+nvidia-nvtx-cu12==12.4.127
+ipython==8.36.0
+opencv-python-headless==4.5.5.64
+MarkupSafe==3.0.2
+jsonschema-specifications==2025.4.1
+wasabi==1.1.3
+blinker==1.9.0
+cfgv==3.4.0
+numpy==2.2.6
+idna==3.10
+nvidia-cufile-cu12==1.11.1.6
+ninja==1.11.1.4
+nvidia-nccl-cu12==2.21.5
+networkx==3.4.2
+certifi==2025.4.26
+deepspeed==0.16.10+b666844f
+pure_eval==0.2.3
+packaging==24.2
+nltk==3.9.1
+contourpy==1.3.2
+pre_commit==4.2.0
+nodeenv==1.9.1
+setuptools==78.1.1
+annotated-types==0.7.0
+multidict==6.4.4
+requests==2.32.3
+tornado==6.5.1
+triton==3.2.0
+pillow==11.2.1
+decord==0.6.0
+shellingham==1.5.4
+streamlit==1.45.1
+pydeck==0.9.1
+confection==0.1.5
+exceptiongroup==1.3.0
+prompt_toolkit==3.0.51
+text-unidecode==1.3
+nvidia-cufft-cu12==11.2.1.3
+antlr4-python3-runtime==4.9.3
+fairscale==0.4.4
+rouge_score==0.1.2
+nvidia-cudnn-cu12==9.1.0.70
+tqdm==4.67.1
+rich==14.0.0
+frozenlist==1.6.0
+webencodings==0.5.1
+altair==5.5.0
+opendatasets==0.1.22
+nvidia-curand-cu12==10.3.5.147
+protobuf==6.31.0
+asttokens==3.0.0
+wheel==0.45.1
+hf-xet==1.1.2
+weasel==0.4.1
+aiosignal==1.3.2
+absl-py==2.2.2
+thinc==8.3.6
+torchvision==0.21.0
+pandas==2.2.3
+fonttools==4.58.0
+bleach==6.2.0
+typing-inspection==0.4.1
+ftfy==6.3.1
+typing_extensions==4.13.2
+nvidia-ml-py==12.575.51
+python-slugify==8.0.4
+lightning-utilities==0.14.3
+py-cpuinfo==9.0.0
+smmap==5.0.2
+regex==2024.11.6
+scikit-image==0.25.2
+iopath==0.1.10
+spacy-legacy==3.0.12
+hjson==3.1.0
+executing==2.2.0
+kiwisolver==1.4.8
+scipy==1.15.3
+aiohappyeyeballs==2.6.1
+toml==0.10.2
+jedi==0.19.2
+GitPython==3.1.44
+ptyprocess==0.7.0
+kaggle==1.7.4.5
+braceexpand==0.1.7
+wcwidth==0.2.13
+nvidia-cuda-runtime-cu12==12.4.127
+pytorch-lightning==2.5.1.post0
+Jinja2==3.1.6
+urllib3==2.4.0
+watchdog==6.0.0
+filelock==3.18.0
+propcache==0.3.1
+torch==2.6.0
+nvidia-cusparse-cu12==12.3.1.170
+cymem==2.0.11
+nvidia-cusolver-cu12==11.6.1.9
+murmurhash==1.0.13
+catalogue==2.0.10
+yarl==1.20.0
+charset-normalizer==3.4.2
+gitdb==4.0.12
+matplotlib==3.10.3
+portalocker==3.1.1
+platformdirs==4.3.8
+async-timeout==5.0.1
+parso==0.8.4
+markdown-it-py==3.0.0
+omegaconf==2.3.0
+cloudpathlib==0.21.1
+nvidia-cusparselt-cu12==0.6.2
+spacy-loggers==1.0.5
+srsly==2.5.1
+identify==2.6.12
+rpds-py==0.25.1
+spacy==3.8.7
+matplotlib-inline==0.1.7
+smart-open==7.1.0
+pydantic==2.11.5
+mdurl==0.1.2
+virtualenv==20.31.2
+pytz==2025.2
+pycocotools==2.0.8
+six==1.17.0
+decorator==5.2.1
+referencing==0.36.2
+sentencepiece==0.2.0
+PyYAML==6.0.2
+pycocoevalcap==1.2
+imageio==2.37.0
+distlib==0.3.9
+pyarrow==20.0.0
+tenacity==9.1.2
+language_data==1.3.0
+nvidia-cuda-cupti-cu12==12.4.127
+blis==1.3.0
+Pygments==2.19.1
+tifffile==2025.5.10
+pyparsing==3.2.3
+cachetools==5.5.2
+safetensors==0.5.3
+attrs==25.3.0
+webdataset==0.2.111
+plotly==6.1.1
+nvidia-cublas-cu12==12.4.5.8
+timm==0.4.12
+torchmetrics==1.7.1
+nvidia-nvjitlink-cu12==12.4.127
+stack-data==0.6.3
+python-dateutil==2.9.0.post0
+lazy_loader==0.4
+traitlets==5.14.3
+einops==0.8.1
+salesforce-lavis==1.0.2
+joblib==1.5.1
+msgpack==1.1.0
+tokenizers==0.21.1
+sentry-sdk==2.29.1
+oss2==2.15.0
+setproctitle==1.3.6
+pip==25.1.1
+cffi==1.17.1
+transformers==4.52.3
+narwhals==1.41.0
+aliyun-python-sdk-core==2.16.0
+jsonschema==4.24.0
+flash-attn==2.7.1.post1
+preshed==3.0.10
+multiprocess==0.70.16
+cryptography==45.0.3
+aliyun-python-sdk-kms==2.16.5
+scikit-learn==1.6.1
+huggingface-hub==0.32.1
+crcmod==1.7
+typer==0.16.0
+web.py==0.62
+docker-pycreds==0.4.0
+xxhash==3.5.0
+bigmodelvis==0.0.1
+datasets==3.6.0
+more-itertools==10.7.0
+yacs==0.1.8
+jmespath==0.10.0
+aiohttp==3.12.2
+opencv-python==4.11.0.86
+pycparser==2.22
+threadpoolctl==3.6.0
+jaraco.functools==4.1.0
+click==8.2.1
+wandb==0.19.11
+opendelta==0.3.2
+pycryptodome==3.23.0
+pathlib==1.0.1
+dill==0.3.8
+fsspec==2025.3.0
+delta-center-client==0.0.4
+cheroot==10.0.1
+typing_extensions==4.12.2
+platformdirs==4.2.2
+jaraco.text==3.12.1
+packaging==24.2
+inflect==7.3.1
+jaraco.context==5.3.0
+wheel==0.45.1
+typeguard==4.3.0
+more-itertools==10.3.0
+tomli==2.0.1
+importlib_metadata==8.0.0
+backports.tarfile==1.2.0
+zipp==3.19.2
+jaraco.collections==5.1.0
+autocommand==2.2.2
+jaraco.functools==4.0.1

ProtT3/all_checkpoints/stage2.5_mol_instruction/wandb/run-20250707_185440-p1815hm9/files/wandb-metadata.json

@@ -0,0 +1,103 @@
+{
+  "os": "Linux-5.10.134-008.16.kangaroo.al8.x86_64-x86_64-with-glibc2.35",
+  "python": "CPython 3.10.0",
+  "startedAt": "2025-07-07T10:54:40.521235Z",
+  "args": [
+    "--devices",
+    "0,1,2,3,4,5,6,7",
+    "--mode",
+    "train",
+    "--filename",
+    "stage2.5_mol_instruction",
+    "--num_query_token",
+    "8",
+    "--save_every_n_epochs",
+    "1",
+    "--max_epochs",
+    "10",
+    "--batch_size",
+    "1",
+    "--precision",
+    "bf16-mixed",
+    "--num_workers",
+    "8",
+    "--plm_model",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m",
+    "--bert_name",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft",
+    "--llm_name",
+    "/oss/wangyujia/BIO/pretrain_output/qwen2.5-7b-instruct-bio/bio_all/save1epoch/checkpoint-1300",
+    "--llm_tune",
+    "mid_lora",
+    "--stage1_path",
+    "/nas/shared/kilab/wangyujia/ProtT3/all_checkpoints/stage1_06290009_deepspeed/epoch=19.ckpt/converted.ckpt",
+    "--use_wandb_logger"
+  ],
+  "program": "/nas/shared/kilab/wangyujia/ProtT3/stage2.py",
+  "codePath": "stage2.py",
+  "email": "gia0603yucca@gmail.com",
+  "root": "./all_checkpoints/stage2.5_mol_instruction/",
+  "host": "dsw-265304-7f6db6b4bb-g4b9r",
+  "executable": "/root/miniconda3/envs/protT3/bin/python",
+  "codePathLocal": "stage2.py",
+  "cpu_count": 64,
+  "cpu_count_logical": 64,
+  "gpu": "NVIDIA A800-SXM4-80GB",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1623302262784",
+      "used": "1260920832"
+    }
+  },
+  "memory": {
+    "total": "549755813888"
+  },
+  "cpu": {
+    "count": 64,
+    "countLogical": 64
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    }
+  ],
+  "cudaVersion": "12.1"
+}

ProtT3/all_checkpoints/stage2.5_mol_instruction/wandb/run-20250707_185440-p1815hm9/logs/debug-internal.log

@@ -0,0 +1,7 @@
+{"time":"2025-07-07T18:54:40.526100632+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.11","symlink path":"all_checkpoints/stage2.5_mol_instruction/wandb/run-20250707_185440-p1815hm9/logs/debug-core.log"}
+{"time":"2025-07-07T18:54:41.559415905+08:00","level":"INFO","msg":"created new stream","id":"p1815hm9"}
+{"time":"2025-07-07T18:54:41.559467213+08:00","level":"INFO","msg":"stream: started","id":"p1815hm9"}
+{"time":"2025-07-07T18:54:41.559497666+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"p1815hm9"}
+{"time":"2025-07-07T18:54:41.559545046+08:00","level":"INFO","msg":"handler: started","stream_id":"p1815hm9"}
+{"time":"2025-07-07T18:54:41.559594522+08:00","level":"INFO","msg":"sender: started","stream_id":"p1815hm9"}
+{"time":"2025-07-07T18:54:42.859018211+08:00","level":"INFO","msg":"Starting system monitor"}

ProtT3/all_checkpoints/stage2.5_mol_instruction/wandb/run-20250707_185440-p1815hm9/logs/debug.log

@@ -0,0 +1,22 @@
+2025-07-07 18:54:40,510 INFO    MainThread:118453 [wandb_setup.py:_flush():70] Current SDK version is 0.19.11
+2025-07-07 18:54:40,510 INFO    MainThread:118453 [wandb_setup.py:_flush():70] Configure stats pid to 118453
+2025-07-07 18:54:40,511 INFO    MainThread:118453 [wandb_setup.py:_flush():70] Loading settings from /root/.config/wandb/settings
+2025-07-07 18:54:40,511 INFO    MainThread:118453 [wandb_setup.py:_flush():70] Loading settings from /nas/shared/kilab/wangyujia/ProtT3/wandb/settings
+2025-07-07 18:54:40,511 INFO    MainThread:118453 [wandb_setup.py:_flush():70] Loading settings from environment variables
+2025-07-07 18:54:40,511 INFO    MainThread:118453 [wandb_init.py:setup_run_log_directory():724] Logging user logs to ./all_checkpoints/stage2.5_mol_instruction/wandb/run-20250707_185440-p1815hm9/logs/debug.log
+2025-07-07 18:54:40,511 INFO    MainThread:118453 [wandb_init.py:setup_run_log_directory():725] Logging internal logs to ./all_checkpoints/stage2.5_mol_instruction/wandb/run-20250707_185440-p1815hm9/logs/debug-internal.log
+2025-07-07 18:54:40,511 INFO    MainThread:118453 [wandb_init.py:init():852] calling init triggers
+2025-07-07 18:54:40,511 INFO    MainThread:118453 [wandb_init.py:init():857] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-07-07 18:54:40,511 INFO    MainThread:118453 [wandb_init.py:init():893] starting backend
+2025-07-07 18:54:40,511 INFO    MainThread:118453 [wandb_init.py:init():897] sending inform_init request
+2025-07-07 18:54:40,512 INFO    MainThread:118453 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-07-07 18:54:40,521 INFO    MainThread:118453 [wandb_init.py:init():907] backend started and connected
+2025-07-07 18:54:40,522 INFO    MainThread:118453 [wandb_init.py:init():1005] updated telemetry
+2025-07-07 18:54:40,523 INFO    MainThread:118453 [wandb_init.py:init():1029] communicating run to backend with 90.0 second timeout
+2025-07-07 18:54:42,809 INFO    MainThread:118453 [wandb_init.py:init():1104] starting run threads in backend
+2025-07-07 18:54:42,991 INFO    MainThread:118453 [wandb_run.py:_console_start():2573] atexit reg
+2025-07-07 18:54:42,991 INFO    MainThread:118453 [wandb_run.py:_redirect():2421] redirect: wrap_raw
+2025-07-07 18:54:42,995 INFO    MainThread:118453 [wandb_run.py:_redirect():2490] Wrapping output streams.
+2025-07-07 18:54:42,999 INFO    MainThread:118453 [wandb_run.py:_redirect():2513] Redirects installed.
+2025-07-07 18:54:43,000 INFO    MainThread:118453 [wandb_init.py:init():1150] run started, returning control to user process

ProtT3/all_checkpoints/stage2.5_mol_instruction/wandb/run-20250707_190145-vu5mgolt/files/output.log

@@ -0,0 +1,4 @@
+/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:654: Checkpoint directory /nas/shared/kilab/wangyujia/ProtT3/all_checkpoints/stage2.5_mol_instruction exists and is not empty.
+Enabling DeepSpeed BF16. Model parameters and inputs will be cast to `bfloat16`.
+LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]
+[rank: 5] Child process with PID 122036 terminated with code 1. Forcefully terminating all other processes to avoid zombies 🧟

ProtT3/all_checkpoints/stage2.5_mol_instruction/wandb/run-20250707_190145-vu5mgolt/files/requirements.txt

@@ -0,0 +1,225 @@
+pydantic_core==2.33.2
+psutil==7.0.0
+nvidia-cuda-nvrtc-cu12==12.4.127
+mpmath==1.3.0
+tzdata==2025.2
+contexttimer==0.3.3
+cycler==0.12.1
+python-magic==0.4.27
+pexpect==4.9.0
+sympy==1.13.1
+wrapt==1.17.2
+marisa-trie==1.2.1
+langcodes==3.5.0
+nvidia-nvtx-cu12==12.4.127
+ipython==8.36.0
+opencv-python-headless==4.5.5.64
+MarkupSafe==3.0.2
+jsonschema-specifications==2025.4.1
+wasabi==1.1.3
+blinker==1.9.0
+cfgv==3.4.0
+numpy==2.2.6
+idna==3.10
+nvidia-cufile-cu12==1.11.1.6
+ninja==1.11.1.4
+nvidia-nccl-cu12==2.21.5
+networkx==3.4.2
+certifi==2025.4.26
+deepspeed==0.16.10+b666844f
+pure_eval==0.2.3
+packaging==24.2
+nltk==3.9.1
+contourpy==1.3.2
+pre_commit==4.2.0
+nodeenv==1.9.1
+setuptools==78.1.1
+annotated-types==0.7.0
+multidict==6.4.4
+requests==2.32.3
+tornado==6.5.1
+triton==3.2.0
+pillow==11.2.1
+decord==0.6.0
+shellingham==1.5.4
+streamlit==1.45.1
+pydeck==0.9.1
+confection==0.1.5
+exceptiongroup==1.3.0
+prompt_toolkit==3.0.51
+text-unidecode==1.3
+nvidia-cufft-cu12==11.2.1.3
+antlr4-python3-runtime==4.9.3
+fairscale==0.4.4
+rouge_score==0.1.2
+nvidia-cudnn-cu12==9.1.0.70
+tqdm==4.67.1
+rich==14.0.0
+frozenlist==1.6.0
+webencodings==0.5.1
+altair==5.5.0
+opendatasets==0.1.22
+nvidia-curand-cu12==10.3.5.147
+protobuf==6.31.0
+asttokens==3.0.0
+wheel==0.45.1
+hf-xet==1.1.2
+weasel==0.4.1
+aiosignal==1.3.2
+absl-py==2.2.2
+thinc==8.3.6
+torchvision==0.21.0
+pandas==2.2.3
+fonttools==4.58.0
+bleach==6.2.0
+typing-inspection==0.4.1
+ftfy==6.3.1
+typing_extensions==4.13.2
+nvidia-ml-py==12.575.51
+python-slugify==8.0.4
+lightning-utilities==0.14.3
+py-cpuinfo==9.0.0
+smmap==5.0.2
+regex==2024.11.6
+scikit-image==0.25.2
+iopath==0.1.10
+spacy-legacy==3.0.12
+hjson==3.1.0
+executing==2.2.0
+kiwisolver==1.4.8
+scipy==1.15.3
+aiohappyeyeballs==2.6.1
+toml==0.10.2
+jedi==0.19.2
+GitPython==3.1.44
+ptyprocess==0.7.0
+kaggle==1.7.4.5
+braceexpand==0.1.7
+wcwidth==0.2.13
+nvidia-cuda-runtime-cu12==12.4.127
+pytorch-lightning==2.5.1.post0
+Jinja2==3.1.6
+urllib3==2.4.0
+watchdog==6.0.0
+filelock==3.18.0
+propcache==0.3.1
+torch==2.6.0
+nvidia-cusparse-cu12==12.3.1.170
+cymem==2.0.11
+nvidia-cusolver-cu12==11.6.1.9
+murmurhash==1.0.13
+catalogue==2.0.10
+yarl==1.20.0
+charset-normalizer==3.4.2
+gitdb==4.0.12
+matplotlib==3.10.3
+portalocker==3.1.1
+platformdirs==4.3.8
+async-timeout==5.0.1
+parso==0.8.4
+markdown-it-py==3.0.0
+omegaconf==2.3.0
+cloudpathlib==0.21.1
+nvidia-cusparselt-cu12==0.6.2
+spacy-loggers==1.0.5
+srsly==2.5.1
+identify==2.6.12
+rpds-py==0.25.1
+spacy==3.8.7
+matplotlib-inline==0.1.7
+smart-open==7.1.0
+pydantic==2.11.5
+mdurl==0.1.2
+virtualenv==20.31.2
+pytz==2025.2
+pycocotools==2.0.8
+six==1.17.0
+decorator==5.2.1
+referencing==0.36.2
+sentencepiece==0.2.0
+PyYAML==6.0.2
+pycocoevalcap==1.2
+imageio==2.37.0
+distlib==0.3.9
+pyarrow==20.0.0
+tenacity==9.1.2
+language_data==1.3.0
+nvidia-cuda-cupti-cu12==12.4.127
+blis==1.3.0
+Pygments==2.19.1
+tifffile==2025.5.10
+pyparsing==3.2.3
+cachetools==5.5.2
+safetensors==0.5.3
+attrs==25.3.0
+webdataset==0.2.111
+plotly==6.1.1
+nvidia-cublas-cu12==12.4.5.8
+timm==0.4.12
+torchmetrics==1.7.1
+nvidia-nvjitlink-cu12==12.4.127
+stack-data==0.6.3
+python-dateutil==2.9.0.post0
+lazy_loader==0.4
+traitlets==5.14.3
+einops==0.8.1
+salesforce-lavis==1.0.2
+joblib==1.5.1
+msgpack==1.1.0
+tokenizers==0.21.1
+sentry-sdk==2.29.1
+oss2==2.15.0
+setproctitle==1.3.6
+pip==25.1.1
+cffi==1.17.1
+transformers==4.52.3
+narwhals==1.41.0
+aliyun-python-sdk-core==2.16.0
+jsonschema==4.24.0
+flash-attn==2.7.1.post1
+preshed==3.0.10
+multiprocess==0.70.16
+cryptography==45.0.3
+aliyun-python-sdk-kms==2.16.5
+scikit-learn==1.6.1
+huggingface-hub==0.32.1
+crcmod==1.7
+typer==0.16.0
+web.py==0.62
+docker-pycreds==0.4.0
+xxhash==3.5.0
+bigmodelvis==0.0.1
+datasets==3.6.0
+more-itertools==10.7.0
+yacs==0.1.8
+jmespath==0.10.0
+aiohttp==3.12.2
+opencv-python==4.11.0.86
+pycparser==2.22
+threadpoolctl==3.6.0
+jaraco.functools==4.1.0
+click==8.2.1
+wandb==0.19.11
+opendelta==0.3.2
+pycryptodome==3.23.0
+pathlib==1.0.1
+dill==0.3.8
+fsspec==2025.3.0
+delta-center-client==0.0.4
+cheroot==10.0.1
+typing_extensions==4.12.2
+platformdirs==4.2.2
+jaraco.text==3.12.1
+packaging==24.2
+inflect==7.3.1
+jaraco.context==5.3.0
+wheel==0.45.1
+typeguard==4.3.0
+more-itertools==10.3.0
+tomli==2.0.1
+importlib_metadata==8.0.0
+backports.tarfile==1.2.0
+zipp==3.19.2
+jaraco.collections==5.1.0
+autocommand==2.2.2
+jaraco.functools==4.0.1