Add files using upload-large-folder tool

BIO/ablation/antibiotic_resistance.jsonl

@@ -0,0 +1,339 @@
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "11", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "7", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "11", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "4", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "11", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "8", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "9", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "8", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "14", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "5", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "5", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "11", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "12", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "12", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "12", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "11", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "14", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "17", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "11", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "11", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "10", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "12", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "14", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "16", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "12", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "5", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "9", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "9", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "9", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "11", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "5", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "4", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "11", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "10", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "10", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "14", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "10", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "12", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "12", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "11", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "10", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "4", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>14</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "11", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "10", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "14", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "11", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "10", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "17", "generated_answer": "<answer>14</answer>"}
+{"reference_answer": "7", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "11", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "10", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "12", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "11", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "5", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "14", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "11", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "11", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "17", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "17", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "5", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "11", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "7", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "10", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "7", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "16", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "11", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "14", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "11", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "10", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "5", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "14", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "4", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "14", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "11", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "10", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "10", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "16", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "12", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "14", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "5", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "5", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "12", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "11", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "7", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "14", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "4", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "12", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "5", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "10", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "5", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "11", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "11", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "11", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "4", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "11", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "5", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "12", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "16", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "10", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "12", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "9", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "10", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "12", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "12", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "7", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "5", "generated_answer": "<answer>14</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "11", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "12", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "12", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "17", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "14", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "9", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "5", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "12", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "9", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "11", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "12", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "9", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "5", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "16", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "12", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "14", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "14", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "5", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "14", "generated_answer": "<answer>14</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "14", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "5", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "4", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "11", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>14</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "12", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "11", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "10", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "12", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "9", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "11", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "7", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "5", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "16", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "4", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "15", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "11", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "5", "generated_answer": "<answer>15</answer>"}
+{"reference_answer": "2", "generated_answer": "<answer>15</answer>"}

BIO/ablation/metal_ion_binding.jsonl

@@ -0,0 +1,718 @@
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>0</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>0</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>0</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>0</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>0</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>0</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>0</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>0</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>0</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>0</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>0</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>0</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>0</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>0</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>0</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>0</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>0</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>0</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>0</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>0</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>0</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>0</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>0</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>0</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>0</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "1", "generated_answer": "<answer>1</answer>"}
+{"reference_answer": "0", "generated_answer": "<answer>1</answer>"}

ProtT3/all_checkpoints/stage2.5_mol_instruction/wandb/run-20250707_200229-yex1pcwt/files/output.log

@@ -0,0 +1,35 @@
+/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:654: Checkpoint directory /nas/shared/kilab/wangyujia/ProtT3/all_checkpoints/stage2.5_mol_instruction exists and is not empty.
+Enabling DeepSpeed BF16. Model parameters and inputs will be cast to `bfloat16`.
+LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]
+
+  | Name  | Type     | Params | Mode
+-------------------------------------------
+0 | blip2 | Blip2OPT | 7.9 B  | train
+-------------------------------------------
+104 M     Trainable params
+7.8 B     Non-trainable params
+7.9 B     Total params
+31,459.025Total estimated model params size (MB)
+174       Modules in train mode
+1203      Modules in eval mode
+Epoch 9: 100%|████████████████████████████████████████████████████| 6572/6572 [22:30<00:00,  4.87it/s, v_num=pcwt]BLEU-2 score: 34.121767513883164
+BLEU-4 score: 29.415650694730406████████████████████████████████████████████| 3287/3287 [4:02:24<00:00,  0.23it/s]
+/nas/shared/kilab/wangyujia/ProtT3/model/dist_funs.py:18: FutureWarning: Positional args are being deprecated, use kwargs instead. Refer to https://pytorch.org/docs/main/generated/torch.nn.Module.html#torch.nn.Module.state_dict for details.
+  sd = self.module.state_dict(destination, prefix, keep_vars)
+105160it [04:48, 364.98it/s]
+105160it [01:55, 912.27it/s]
+/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/connectors/logger_connector/result.py:434: It is recommended to use `self.log('dataset0/acc', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices.
+Average Meteor score: 58.413905959274196
+/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/connectors/logger_connector/result.py:434: It is recommended to use `self.log('dataset0/bleu2', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices.
+ROUGE score:
+rouge1: 46.26000494549008
+rouge2: 35.3808428047301
+rougeL: 39.73314434393358
+/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/connectors/logger_connector/result.py:434: It is recommended to use `self.log('dataset0/bleu4', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices.
+/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/connectors/logger_connector/result.py:434: It is recommended to use `self.log('dataset0/rouge_1', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices.
+/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/connectors/logger_connector/result.py:434: It is recommended to use `self.log('dataset0/rouge_2', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices.
+/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/connectors/logger_connector/result.py:434: It is recommended to use `self.log('dataset0/rouge_l', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices.
+/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/connectors/logger_connector/result.py:434: It is recommended to use `self.log('dataset0/meteor_score', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices.
+Epoch 9: 100%|██████████████████████████████████████████████████| 6572/6572 [4:42:04<00:00,  0.39it/s, v_num=pcwt]
+
+`Trainer.fit` stopped: `max_epochs=10` reached.

ProtT3/all_checkpoints/stage2.5_mol_instruction/wandb/run-20250707_200229-yex1pcwt/files/requirements.txt

@@ -0,0 +1,225 @@
+pydantic_core==2.33.2
+psutil==7.0.0
+nvidia-cuda-nvrtc-cu12==12.4.127
+mpmath==1.3.0
+tzdata==2025.2
+contexttimer==0.3.3
+cycler==0.12.1
+python-magic==0.4.27
+pexpect==4.9.0
+sympy==1.13.1
+wrapt==1.17.2
+marisa-trie==1.2.1
+langcodes==3.5.0
+nvidia-nvtx-cu12==12.4.127
+ipython==8.36.0
+opencv-python-headless==4.5.5.64
+MarkupSafe==3.0.2
+jsonschema-specifications==2025.4.1
+wasabi==1.1.3
+blinker==1.9.0
+cfgv==3.4.0
+numpy==2.2.6
+idna==3.10
+nvidia-cufile-cu12==1.11.1.6
+ninja==1.11.1.4
+nvidia-nccl-cu12==2.21.5
+networkx==3.4.2
+certifi==2025.4.26
+deepspeed==0.16.10+b666844f
+pure_eval==0.2.3
+packaging==24.2
+nltk==3.9.1
+contourpy==1.3.2
+pre_commit==4.2.0
+nodeenv==1.9.1
+setuptools==78.1.1
+annotated-types==0.7.0
+multidict==6.4.4
+requests==2.32.3
+tornado==6.5.1
+triton==3.2.0
+pillow==11.2.1
+decord==0.6.0
+shellingham==1.5.4
+streamlit==1.45.1
+pydeck==0.9.1
+confection==0.1.5
+exceptiongroup==1.3.0
+prompt_toolkit==3.0.51
+text-unidecode==1.3
+nvidia-cufft-cu12==11.2.1.3
+antlr4-python3-runtime==4.9.3
+fairscale==0.4.4
+rouge_score==0.1.2
+nvidia-cudnn-cu12==9.1.0.70
+tqdm==4.67.1
+rich==14.0.0
+frozenlist==1.6.0
+webencodings==0.5.1
+altair==5.5.0
+opendatasets==0.1.22
+nvidia-curand-cu12==10.3.5.147
+protobuf==6.31.0
+asttokens==3.0.0
+wheel==0.45.1
+hf-xet==1.1.2
+weasel==0.4.1
+aiosignal==1.3.2
+absl-py==2.2.2
+thinc==8.3.6
+torchvision==0.21.0
+pandas==2.2.3
+fonttools==4.58.0
+bleach==6.2.0
+typing-inspection==0.4.1
+ftfy==6.3.1
+typing_extensions==4.13.2
+nvidia-ml-py==12.575.51
+python-slugify==8.0.4
+lightning-utilities==0.14.3
+py-cpuinfo==9.0.0
+smmap==5.0.2
+regex==2024.11.6
+scikit-image==0.25.2
+iopath==0.1.10
+spacy-legacy==3.0.12
+hjson==3.1.0
+executing==2.2.0
+kiwisolver==1.4.8
+scipy==1.15.3
+aiohappyeyeballs==2.6.1
+toml==0.10.2
+jedi==0.19.2
+GitPython==3.1.44
+ptyprocess==0.7.0
+kaggle==1.7.4.5
+braceexpand==0.1.7
+wcwidth==0.2.13
+nvidia-cuda-runtime-cu12==12.4.127
+pytorch-lightning==2.5.1.post0
+Jinja2==3.1.6
+urllib3==2.4.0
+watchdog==6.0.0
+filelock==3.18.0
+propcache==0.3.1
+torch==2.6.0
+nvidia-cusparse-cu12==12.3.1.170
+cymem==2.0.11
+nvidia-cusolver-cu12==11.6.1.9
+murmurhash==1.0.13
+catalogue==2.0.10
+yarl==1.20.0
+charset-normalizer==3.4.2
+gitdb==4.0.12
+matplotlib==3.10.3
+portalocker==3.1.1
+platformdirs==4.3.8
+async-timeout==5.0.1
+parso==0.8.4
+markdown-it-py==3.0.0
+omegaconf==2.3.0
+cloudpathlib==0.21.1
+nvidia-cusparselt-cu12==0.6.2
+spacy-loggers==1.0.5
+srsly==2.5.1
+identify==2.6.12
+rpds-py==0.25.1
+spacy==3.8.7
+matplotlib-inline==0.1.7
+smart-open==7.1.0
+pydantic==2.11.5
+mdurl==0.1.2
+virtualenv==20.31.2
+pytz==2025.2
+pycocotools==2.0.8
+six==1.17.0
+decorator==5.2.1
+referencing==0.36.2
+sentencepiece==0.2.0
+PyYAML==6.0.2
+pycocoevalcap==1.2
+imageio==2.37.0
+distlib==0.3.9
+pyarrow==20.0.0
+tenacity==9.1.2
+language_data==1.3.0
+nvidia-cuda-cupti-cu12==12.4.127
+blis==1.3.0
+Pygments==2.19.1
+tifffile==2025.5.10
+pyparsing==3.2.3
+cachetools==5.5.2
+safetensors==0.5.3
+attrs==25.3.0
+webdataset==0.2.111
+plotly==6.1.1
+nvidia-cublas-cu12==12.4.5.8
+timm==0.4.12
+torchmetrics==1.7.1
+nvidia-nvjitlink-cu12==12.4.127
+stack-data==0.6.3
+python-dateutil==2.9.0.post0
+lazy_loader==0.4
+traitlets==5.14.3
+einops==0.8.1
+salesforce-lavis==1.0.2
+joblib==1.5.1
+msgpack==1.1.0
+tokenizers==0.21.1
+sentry-sdk==2.29.1
+oss2==2.15.0
+setproctitle==1.3.6
+pip==25.1.1
+cffi==1.17.1
+transformers==4.52.3
+narwhals==1.41.0
+aliyun-python-sdk-core==2.16.0
+jsonschema==4.24.0
+flash-attn==2.7.1.post1
+preshed==3.0.10
+multiprocess==0.70.16
+cryptography==45.0.3
+aliyun-python-sdk-kms==2.16.5
+scikit-learn==1.6.1
+huggingface-hub==0.32.1
+crcmod==1.7
+typer==0.16.0
+web.py==0.62
+docker-pycreds==0.4.0
+xxhash==3.5.0
+bigmodelvis==0.0.1
+datasets==3.6.0
+more-itertools==10.7.0
+yacs==0.1.8
+jmespath==0.10.0
+aiohttp==3.12.2
+opencv-python==4.11.0.86
+pycparser==2.22
+threadpoolctl==3.6.0
+jaraco.functools==4.1.0
+click==8.2.1
+wandb==0.19.11
+opendelta==0.3.2
+pycryptodome==3.23.0
+pathlib==1.0.1
+dill==0.3.8
+fsspec==2025.3.0
+delta-center-client==0.0.4
+cheroot==10.0.1
+typing_extensions==4.12.2
+platformdirs==4.2.2
+jaraco.text==3.12.1
+packaging==24.2
+inflect==7.3.1
+jaraco.context==5.3.0
+wheel==0.45.1
+typeguard==4.3.0
+more-itertools==10.3.0
+tomli==2.0.1
+importlib_metadata==8.0.0
+backports.tarfile==1.2.0
+zipp==3.19.2
+jaraco.collections==5.1.0
+autocommand==2.2.2
+jaraco.functools==4.0.1

ProtT3/all_checkpoints/stage2.5_mol_instruction/wandb/run-20250707_200229-yex1pcwt/files/wandb-metadata.json

@@ -0,0 +1,103 @@
+{
+  "os": "Linux-5.10.134-008.16.kangaroo.al8.x86_64-x86_64-with-glibc2.35",
+  "python": "CPython 3.10.0",
+  "startedAt": "2025-07-07T12:02:29.753102Z",
+  "args": [
+    "--devices",
+    "0,1,2,3,4,5,6,7",
+    "--mode",
+    "train",
+    "--filename",
+    "stage2.5_mol_instruction",
+    "--num_query_token",
+    "8",
+    "--save_every_n_epochs",
+    "1",
+    "--max_epochs",
+    "10",
+    "--batch_size",
+    "2",
+    "--precision",
+    "bf16-mixed",
+    "--num_workers",
+    "8",
+    "--plm_model",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m",
+    "--bert_name",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft",
+    "--llm_name",
+    "/oss/wangyujia/BIO/pretrain_output/qwen2.5-7b-instruct-bio/bio_all/save1epoch/checkpoint-1300",
+    "--llm_tune",
+    "mid_lora",
+    "--stage1_path",
+    "/nas/shared/kilab/wangyujia/ProtT3/all_checkpoints/stage1_06290009_deepspeed/epoch=19.ckpt/converted.ckpt",
+    "--use_wandb_logger"
+  ],
+  "program": "/nas/shared/kilab/wangyujia/ProtT3/stage2.py",
+  "codePath": "stage2.py",
+  "email": "gia0603yucca@gmail.com",
+  "root": "./all_checkpoints/stage2.5_mol_instruction/",
+  "host": "dsw-265304-7f6db6b4bb-g4b9r",
+  "executable": "/root/miniconda3/envs/protT3/bin/python",
+  "codePathLocal": "stage2.py",
+  "cpu_count": 64,
+  "cpu_count_logical": 64,
+  "gpu": "NVIDIA A800-SXM4-80GB",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1623302262784",
+      "used": "1260978176"
+    }
+  },
+  "memory": {
+    "total": "549755813888"
+  },
+  "cpu": {
+    "count": 64,
+    "countLogical": 64
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    }
+  ],
+  "cudaVersion": "12.1"
+}

ProtT3/all_checkpoints/stage2.5_mol_instruction/wandb/run-20250707_200229-yex1pcwt/files/wandb-summary.json

@@ -0,0 +1 @@
+{"trainer/global_step":65719,"dataset0/rouge_l":39.7331428527832,"_runtime":33448.966741938,"_wandb":{"runtime":33462},"epoch":9,"dataset0/rouge_2":35.38084411621094,"loss":0.07072551548480988,"dataset0/bleu2":34.121768951416016,"dataset0/rouge_1":46.260005950927734,"dataloader0/val loss/dataloader_idx_0":0.18945328891277313,"dataset0/meteor_score":58.41390609741211,"lr":1.2202456673549023e-05,"_step":1323,"dataset0/acc":0,"_timestamp":1.7519231987190707e+09,"dataset0/bleu4":29.415651321411133}

ProtT3/all_checkpoints/stage2.5_mol_instruction/wandb/run-20250707_200229-yex1pcwt/logs/debug-internal.log

@@ -0,0 +1,17 @@
+{"time":"2025-07-07T20:02:29.75666986+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.11","symlink path":"all_checkpoints/stage2.5_mol_instruction/wandb/run-20250707_200229-yex1pcwt/logs/debug-core.log"}
+{"time":"2025-07-07T20:02:30.922635932+08:00","level":"INFO","msg":"created new stream","id":"yex1pcwt"}
+{"time":"2025-07-07T20:02:30.922678667+08:00","level":"INFO","msg":"stream: started","id":"yex1pcwt"}
+{"time":"2025-07-07T20:02:30.922713833+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"yex1pcwt"}
+{"time":"2025-07-07T20:02:30.922757284+08:00","level":"INFO","msg":"sender: started","stream_id":"yex1pcwt"}
+{"time":"2025-07-07T20:02:30.92278615+08:00","level":"INFO","msg":"handler: started","stream_id":"yex1pcwt"}
+{"time":"2025-07-07T20:02:32.296458789+08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2025-07-08T01:01:50.39071972+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": read tcp 10.1.4.159:39416->104.21.20.172:443: read: connection reset by peer"}
+{"time":"2025-07-08T01:07:45.887474022+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2.5_mol_instruction/yex1pcwt/file_stream\": read tcp 10.1.4.159:48838->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-08T05:20:12.207426797+08:00","level":"INFO","msg":"stream: closing","id":"yex1pcwt"}
+{"time":"2025-07-08T05:20:12.207468139+08:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2025-07-08T05:20:12.208684636+08:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2025-07-08T05:20:13.938647534+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-07-08T05:20:14.977621841+08:00","level":"INFO","msg":"handler: closed","stream_id":"yex1pcwt"}
+{"time":"2025-07-08T05:20:14.977653692+08:00","level":"INFO","msg":"sender: closed","stream_id":"yex1pcwt"}
+{"time":"2025-07-08T05:20:14.977651902+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"yex1pcwt"}
+{"time":"2025-07-08T05:20:14.982274952+08:00","level":"INFO","msg":"stream: closed","id":"yex1pcwt"}

ProtT3/all_checkpoints/stage2.5_mol_instruction/wandb/run-20250707_200229-yex1pcwt/logs/debug.log

@@ -0,0 +1,24 @@
+2025-07-07 20:02:29,743 INFO    MainThread:129761 [wandb_setup.py:_flush():70] Current SDK version is 0.19.11
+2025-07-07 20:02:29,743 INFO    MainThread:129761 [wandb_setup.py:_flush():70] Configure stats pid to 129761
+2025-07-07 20:02:29,743 INFO    MainThread:129761 [wandb_setup.py:_flush():70] Loading settings from /root/.config/wandb/settings
+2025-07-07 20:02:29,743 INFO    MainThread:129761 [wandb_setup.py:_flush():70] Loading settings from /nas/shared/kilab/wangyujia/ProtT3/wandb/settings
+2025-07-07 20:02:29,743 INFO    MainThread:129761 [wandb_setup.py:_flush():70] Loading settings from environment variables
+2025-07-07 20:02:29,743 INFO    MainThread:129761 [wandb_init.py:setup_run_log_directory():724] Logging user logs to ./all_checkpoints/stage2.5_mol_instruction/wandb/run-20250707_200229-yex1pcwt/logs/debug.log
+2025-07-07 20:02:29,743 INFO    MainThread:129761 [wandb_init.py:setup_run_log_directory():725] Logging internal logs to ./all_checkpoints/stage2.5_mol_instruction/wandb/run-20250707_200229-yex1pcwt/logs/debug-internal.log
+2025-07-07 20:02:29,743 INFO    MainThread:129761 [wandb_init.py:init():852] calling init triggers
+2025-07-07 20:02:29,743 INFO    MainThread:129761 [wandb_init.py:init():857] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-07-07 20:02:29,743 INFO    MainThread:129761 [wandb_init.py:init():893] starting backend
+2025-07-07 20:02:29,743 INFO    MainThread:129761 [wandb_init.py:init():897] sending inform_init request
+2025-07-07 20:02:29,745 INFO    MainThread:129761 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-07-07 20:02:29,748 INFO    MainThread:129761 [wandb_init.py:init():907] backend started and connected
+2025-07-07 20:02:29,754 INFO    MainThread:129761 [wandb_init.py:init():1005] updated telemetry
+2025-07-07 20:02:29,762 INFO    MainThread:129761 [wandb_init.py:init():1029] communicating run to backend with 90.0 second timeout
+2025-07-07 20:02:32,258 INFO    MainThread:129761 [wandb_init.py:init():1104] starting run threads in backend
+2025-07-07 20:02:32,427 INFO    MainThread:129761 [wandb_run.py:_console_start():2573] atexit reg
+2025-07-07 20:02:32,427 INFO    MainThread:129761 [wandb_run.py:_redirect():2421] redirect: wrap_raw
+2025-07-07 20:02:32,457 INFO    MainThread:129761 [wandb_run.py:_redirect():2490] Wrapping output streams.
+2025-07-07 20:02:32,462 INFO    MainThread:129761 [wandb_run.py:_redirect():2513] Redirects installed.
+2025-07-07 20:02:32,463 INFO    MainThread:129761 [wandb_init.py:init():1150] run started, returning control to user process
+2025-07-07 20:02:40,689 INFO    MainThread:129761 [wandb_run.py:_config_callback():1436] config_cb None None {'filename': 'stage2.5_mol_instruction', 'seed': 42, 'mode': 'train', 'strategy': 'deepspeed', 'accelerator': 'gpu', 'devices': '0,1,2,3,4,5,6,7', 'precision': 'bf16-mixed', 'max_epochs': 10, 'accumulate_grad_batches': 1, 'check_val_every_n_epoch': 1, 'enable_flash': False, 'use_wandb_logger': True, 'mix_dataset': False, 'save_every_n_epochs': 1, 'bert_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft', 'cross_attention_freq': 2, 'num_query_token': 8, 'llm_name': '/oss/wangyujia/BIO/pretrain_output/qwen2.5-7b-instruct-bio/bio_all/save1epoch/checkpoint-1300', 'num_beams': 5, 'do_sample': False, 'max_inference_len': 128, 'min_inference_len': 1, 'llm_tune': 'mid_lora', 'peft_config': '', 'peft_dir': '', 'plm_model': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m', 'plm_tune': 'freeze', 'lora_r': 8, 'lora_alpha': 16, 'lora_dropout': 0.1, 'enbale_gradient_checkpointing': False, 'weight_decay': 0.05, 'init_lr': 0.0001, 'min_lr': 1e-05, 'warmup_lr': 1e-06, 'warmup_steps': 1000, 'lr_decay_rate': 0.9, 'scheduler': 'linear_warmup_cosine_lr', 'stage1_path': '/nas/shared/kilab/wangyujia/ProtT3/all_checkpoints/stage1_06290009_deepspeed/epoch=19.ckpt/converted.ckpt', 'stage2_path': '', 'init_checkpoint': '', 'caption_eval_epoch': 10, 'num_workers': 8, 'batch_size': 2, 'inference_batch_size': 4, 'root': 'data', 'text_max_len': 1024, 'q_max_len': 29, 'a_max_len': 36, 'prot_max_len': 1024, 'prompt': 'The protein has the following properties: ', 'filter_side_qa': False}
+2025-07-08 05:20:12,205 INFO    MsgRouterThr:129761 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles.

ProtT3/all_checkpoints/stage2_07021249/wandb/debug-internal.log

@@ -0,0 +1,17 @@
+{"time":"2025-07-02T13:01:02.517398459+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.11","symlink path":"all_checkpoints/stage2_07021249/wandb/run-20250702_130101-ttc1macp/logs/debug-core.log"}
+{"time":"2025-07-02T13:01:03.639276653+08:00","level":"INFO","msg":"created new stream","id":"ttc1macp"}
+{"time":"2025-07-02T13:01:03.63932174+08:00","level":"INFO","msg":"stream: started","id":"ttc1macp"}
+{"time":"2025-07-02T13:01:03.639331986+08:00","level":"INFO","msg":"handler: started","stream_id":"ttc1macp"}
+{"time":"2025-07-02T13:01:03.639357726+08:00","level":"INFO","msg":"sender: started","stream_id":"ttc1macp"}
+{"time":"2025-07-02T13:01:03.639363515+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"ttc1macp"}
+{"time":"2025-07-02T13:01:05.044613784+08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2025-07-02T14:25:40.366902458+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07021249/ttc1macp/file_stream\": unexpected EOF"}
+{"time":"2025-07-02T15:45:50.370249421+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-07-02T17:37:30.121166513+08:00","level":"INFO","msg":"stream: closing","id":"ttc1macp"}
+{"time":"2025-07-02T17:37:30.121209819+08:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2025-07-02T17:37:30.137879266+08:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2025-07-02T17:39:43.997948982+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-07-02T17:39:45.216379261+08:00","level":"INFO","msg":"handler: closed","stream_id":"ttc1macp"}
+{"time":"2025-07-02T17:39:45.216437206+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"ttc1macp"}
+{"time":"2025-07-02T17:39:45.216450748+08:00","level":"INFO","msg":"sender: closed","stream_id":"ttc1macp"}
+{"time":"2025-07-02T17:39:45.269415307+08:00","level":"INFO","msg":"stream: closed","id":"ttc1macp"}

ProtT3/all_checkpoints/stage2_07021249/wandb/debug.log

@@ -0,0 +1,24 @@
+2025-07-02 13:01:02,446 INFO    MainThread:2341 [wandb_setup.py:_flush():70] Current SDK version is 0.19.11
+2025-07-02 13:01:02,446 INFO    MainThread:2341 [wandb_setup.py:_flush():70] Configure stats pid to 2341
+2025-07-02 13:01:02,446 INFO    MainThread:2341 [wandb_setup.py:_flush():70] Loading settings from /root/.config/wandb/settings
+2025-07-02 13:01:02,446 INFO    MainThread:2341 [wandb_setup.py:_flush():70] Loading settings from /nas/shared/kilab/wangyujia/ProtT3/wandb/settings
+2025-07-02 13:01:02,446 INFO    MainThread:2341 [wandb_setup.py:_flush():70] Loading settings from environment variables
+2025-07-02 13:01:02,446 INFO    MainThread:2341 [wandb_init.py:setup_run_log_directory():724] Logging user logs to ./all_checkpoints/stage2_07021249/wandb/run-20250702_130101-ttc1macp/logs/debug.log
+2025-07-02 13:01:02,446 INFO    MainThread:2341 [wandb_init.py:setup_run_log_directory():725] Logging internal logs to ./all_checkpoints/stage2_07021249/wandb/run-20250702_130101-ttc1macp/logs/debug-internal.log
+2025-07-02 13:01:02,446 INFO    MainThread:2341 [wandb_init.py:init():852] calling init triggers
+2025-07-02 13:01:02,446 INFO    MainThread:2341 [wandb_init.py:init():857] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-07-02 13:01:02,446 INFO    MainThread:2341 [wandb_init.py:init():893] starting backend
+2025-07-02 13:01:02,446 INFO    MainThread:2341 [wandb_init.py:init():897] sending inform_init request
+2025-07-02 13:01:02,449 INFO    MainThread:2341 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-07-02 13:01:02,449 INFO    MainThread:2341 [wandb_init.py:init():907] backend started and connected
+2025-07-02 13:01:02,468 INFO    MainThread:2341 [wandb_init.py:init():1005] updated telemetry
+2025-07-02 13:01:02,498 INFO    MainThread:2341 [wandb_init.py:init():1029] communicating run to backend with 90.0 second timeout
+2025-07-02 13:01:04,915 INFO    MainThread:2341 [wandb_init.py:init():1104] starting run threads in backend
+2025-07-02 13:01:05,214 INFO    MainThread:2341 [wandb_run.py:_console_start():2573] atexit reg
+2025-07-02 13:01:05,215 INFO    MainThread:2341 [wandb_run.py:_redirect():2421] redirect: wrap_raw
+2025-07-02 13:01:05,215 INFO    MainThread:2341 [wandb_run.py:_redirect():2490] Wrapping output streams.
+2025-07-02 13:01:05,215 INFO    MainThread:2341 [wandb_run.py:_redirect():2513] Redirects installed.
+2025-07-02 13:01:05,218 INFO    MainThread:2341 [wandb_init.py:init():1150] run started, returning control to user process
+2025-07-02 13:01:12,114 INFO    MainThread:2341 [wandb_run.py:_config_callback():1436] config_cb None None {'filename': 'stage2_07021249', 'seed': 42, 'mode': 'train', 'strategy': 'deepspeed', 'accelerator': 'gpu', 'devices': '0,1,2,3,4,5,6,7', 'precision': 'bf16-mixed', 'max_epochs': 10, 'accumulate_grad_batches': 1, 'check_val_every_n_epoch': 1, 'enable_flash': False, 'use_wandb_logger': True, 'mix_dataset': True, 'save_every_n_epochs': 5, 'bert_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft', 'cross_attention_freq': 2, 'num_query_token': 8, 'llm_name': '/oss/wangyujia/BIO/pretrain_output/qwen2.5-7b-instruct-bio/bio_all/save1epoch/checkpoint-1300', 'num_beams': 5, 'do_sample': False, 'max_inference_len': 128, 'min_inference_len': 1, 'llm_tune': 'mid_lora', 'peft_config': '', 'peft_dir': '', 'plm_model': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m', 'plm_tune': 'freeze', 'lora_r': 8, 'lora_alpha': 16, 'lora_dropout': 0.1, 'enbale_gradient_checkpointing': False, 'weight_decay': 0.05, 'init_lr': 0.0001, 'min_lr': 1e-05, 'warmup_lr': 1e-06, 'warmup_steps': 1000, 'lr_decay_rate': 0.9, 'scheduler': 'linear_warmup_cosine_lr', 'stage1_path': '/nas/shared/kilab/wangyujia/ProtT3/all_checkpoints/stage1_06290009_deepspeed/epoch=19.ckpt/converted.ckpt', 'stage2_path': '', 'init_checkpoint': '', 'caption_eval_epoch': 10, 'num_workers': 8, 'batch_size': 32, 'inference_batch_size': 4, 'root': 'data', 'text_max_len': 128, 'q_max_len': 29, 'a_max_len': 36, 'prot_max_len': 1024, 'prompt': 'The protein has the following properties: ', 'filter_side_qa': False}
+2025-07-02 17:37:30,072 INFO    MsgRouterThr:2341 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles.

ProtT3/all_checkpoints/stage2_07021249/wandb/run-20250702_130101-ttc1macp/files/config.yaml

@@ -0,0 +1,222 @@
+_wandb:
+    value:
+        cli_version: 0.19.11
+        m:
+            - "1": dataset0/rouge_l
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": trainer/global_step
+              "6":
+                - 3
+              "7": []
+            - "1": lr
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": epoch
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": dataset0/rouge_1
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": dataset0/rouge_2
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": dataset0/meteor_score
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": dataset0/acc
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": dataset0/bleu4
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loss
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": dataloader2/val loss/dataloader_idx_2
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": dataloader0/val loss/dataloader_idx_0
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": dataset0/bleu2
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+        python_version: 3.10.0
+        t:
+            "1":
+                - 1
+                - 5
+                - 9
+                - 11
+                - 33
+                - 41
+                - 49
+                - 53
+                - 55
+                - 63
+                - 103
+            "2":
+                - 1
+                - 5
+                - 9
+                - 11
+                - 33
+                - 41
+                - 49
+                - 53
+                - 55
+                - 63
+                - 103
+            "3":
+                - 7
+                - 23
+                - 55
+                - 66
+            "4": 3.10.0
+            "5": 0.19.11
+            "6": 4.52.3
+            "8":
+                - 5
+            "12": 0.19.11
+            "13": linux-x86_64
+a_max_len:
+    value: 36
+accelerator:
+    value: gpu
+accumulate_grad_batches:
+    value: 1
+batch_size:
+    value: 32
+bert_name:
+    value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft
+caption_eval_epoch:
+    value: 10
+check_val_every_n_epoch:
+    value: 1
+cross_attention_freq:
+    value: 2
+devices:
+    value: 0,1,2,3,4,5,6,7
+do_sample:
+    value: false
+enable_flash:
+    value: false
+enbale_gradient_checkpointing:
+    value: false
+filename:
+    value: stage2_07021249
+filter_side_qa:
+    value: false
+inference_batch_size:
+    value: 4
+init_checkpoint:
+    value: ""
+init_lr:
+    value: 0.0001
+llm_name:
+    value: /oss/wangyujia/BIO/pretrain_output/qwen2.5-7b-instruct-bio/bio_all/save1epoch/checkpoint-1300
+llm_tune:
+    value: mid_lora
+lora_alpha:
+    value: 16
+lora_dropout:
+    value: 0.1
+lora_r:
+    value: 8
+lr_decay_rate:
+    value: 0.9
+max_epochs:
+    value: 10
+max_inference_len:
+    value: 128
+min_inference_len:
+    value: 1
+min_lr:
+    value: 1e-05
+mix_dataset:
+    value: true
+mode:
+    value: train
+num_beams:
+    value: 5
+num_query_token:
+    value: 8
+num_workers:
+    value: 8
+peft_config:
+    value: ""
+peft_dir:
+    value: ""
+plm_model:
+    value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m
+plm_tune:
+    value: freeze
+precision:
+    value: bf16-mixed
+prompt:
+    value: 'The protein has the following properties: '
+prot_max_len:
+    value: 1024
+q_max_len:
+    value: 29
+root:
+    value: data
+save_every_n_epochs:
+    value: 5
+scheduler:
+    value: linear_warmup_cosine_lr
+seed:
+    value: 42
+stage1_path:
+    value: /nas/shared/kilab/wangyujia/ProtT3/all_checkpoints/stage1_06290009_deepspeed/epoch=19.ckpt/converted.ckpt
+stage2_path:
+    value: ""
+strategy:
+    value: deepspeed
+text_max_len:
+    value: 128
+use_wandb_logger:
+    value: true
+warmup_lr:
+    value: 1e-06
+warmup_steps:
+    value: 1000
+weight_decay:
+    value: 0.05

ProtT3/all_checkpoints/stage2_07021249/wandb/run-20250702_130101-ttc1macp/files/output.log

@@ -0,0 +1,35 @@
+/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:654: Checkpoint directory /nas/shared/kilab/wangyujia/ProtT3/all_checkpoints/stage2_07021249 exists and is not empty.
+Enabling DeepSpeed BF16. Model parameters and inputs will be cast to `bfloat16`.
+LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]
+
+  | Name  | Type     | Params | Mode
+-------------------------------------------
+0 | blip2 | Blip2OPT | 3.5 B  | train
+-------------------------------------------
+110 M     Trainable params
+3.4 B     Non-trainable params
+3.5 B     Total params
+14,075.005Total estimated model params size (MB)
+342       Modules in train mode
+1148      Modules in eval mode
+Epoch 9: 100%|██████████████████████████████████████████| 1682/1682 [22:58<00:00,  1.22it/s, v_num=macp]BLEU-2 score: 15.341177980749976
+BLEU-4 score: 11.860699475707873██████████████████████████████████████| 313/313 [17:47<00:00,  0.29it/s]                          
+/nas/shared/kilab/wangyujia/ProtT3/model/dist_funs.py:18: FutureWarning: Positional args are being deprecated, use kwargs instead. Refer to https://pytorch.org/docs/main/generated/torch.nn.Module.html#torch.nn.Module.state_dict for details.
+  sd = self.module.state_dict(destination, prefix, keep_vars)
+20000it [01:41, 197.92it/s]
+20000it [00:24, 802.60it/s]
+/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/connectors/logger_connector/result.py:434: It is recommended to use `self.log('dataset0/acc', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices.
+Average Meteor score: 26.00935336031241
+/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/connectors/logger_connector/result.py:434: It is recommended to use `self.log('dataset0/bleu2', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices.
+ROUGE score:
+rouge1: 21.952246333483618
+rouge2: 13.41357541493452
+rougeL: 19.765010605045184
+/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/connectors/logger_connector/result.py:434: It is recommended to use `self.log('dataset0/bleu4', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices.
+/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/connectors/logger_connector/result.py:434: It is recommended to use `self.log('dataset0/rouge_1', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices.
+/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/connectors/logger_connector/result.py:434: It is recommended to use `self.log('dataset0/rouge_2', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices.
+/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/connectors/logger_connector/result.py:434: It is recommended to use `self.log('dataset0/rouge_l', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices.
+/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/connectors/logger_connector/result.py:434: It is recommended to use `self.log('dataset0/meteor_score', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices.
+Epoch 9: 100%|████████████████████████████████████████| 1682/1682 [1:01:47<00:00,  0.45it/s, v_num=macp]
+
+`Trainer.fit` stopped: `max_epochs=10` reached.

ProtT3/all_checkpoints/stage2_07021249/wandb/run-20250702_130101-ttc1macp/files/requirements.txt

@@ -0,0 +1,225 @@
+stack-data==0.6.3
+yarl==1.20.0
+setuptools==78.1.1
+cloudpathlib==0.21.1
+pytz==2025.2
+nvidia-cufile-cu12==1.11.1.6
+shellingham==1.5.4
+nvidia-cusolver-cu12==11.6.1.9
+Jinja2==3.1.6
+pycocotools==2.0.8
+pandas==2.2.3
+scipy==1.15.3
+tenacity==9.1.2
+lightning-utilities==0.14.3
+cfgv==3.4.0
+hf-xet==1.1.2
+platformdirs==4.3.8
+smart-open==7.1.0
+text-unidecode==1.3
+nvidia-cublas-cu12==12.4.5.8
+omegaconf==2.3.0
+referencing==0.36.2
+mdurl==0.1.2
+gitdb==4.0.12
+identify==2.6.12
+ipython==8.36.0
+spacy-loggers==1.0.5
+distlib==0.3.9
+typing-inspection==0.4.1
+antlr4-python3-runtime==4.9.3
+multidict==6.4.4
+nvidia-curand-cu12==10.3.5.147
+prompt_toolkit==3.0.51
+Pygments==2.19.1
+numpy==2.2.6
+decord==0.6.0
+srsly==2.5.1
+watchdog==6.0.0
+pure_eval==0.2.3
+virtualenv==20.31.2
+altair==5.5.0
+matplotlib-inline==0.1.7
+bleach==6.2.0
+exceptiongroup==1.3.0
+fairscale==0.4.4
+confection==0.1.5
+fonttools==4.58.0
+nvidia-cuda-nvrtc-cu12==12.4.127
+ptyprocess==0.7.0
+pytorch-lightning==2.5.1.post0
+nodeenv==1.9.1
+nvidia-cudnn-cu12==9.1.0.70
+requests==2.32.3
+marisa-trie==1.2.1
+cachetools==5.5.2
+matplotlib==3.10.3
+typing_extensions==4.13.2
+asttokens==3.0.0
+torch==2.6.0
+PyYAML==6.0.2
+tifffile==2025.5.10
+spacy==3.8.7
+braceexpand==0.1.7
+plotly==6.1.1
+attrs==25.3.0
+py-cpuinfo==9.0.0
+frozenlist==1.6.0
+catalogue==2.0.10
+nvidia-cusparselt-cu12==0.6.2
+traitlets==5.14.3
+annotated-types==0.7.0
+language_data==1.3.0
+thinc==8.3.6
+imageio==2.37.0
+nvidia-cuda-runtime-cu12==12.4.127
+certifi==2025.4.26
+smmap==5.0.2
+python-magic==0.4.27
+triton==3.2.0
+weasel==0.4.1
+async-timeout==5.0.1
+wcwidth==0.2.13
+pillow==11.2.1
+torchmetrics==1.7.1
+kaggle==1.7.4.5
+regex==2024.11.6
+aiosignal==1.3.2
+nvidia-cusparse-cu12==12.3.1.170
+scikit-image==0.25.2
+nvidia-nvtx-cu12==12.4.127
+opendatasets==0.1.22
+iopath==0.1.10
+pyparsing==3.2.3
+portalocker==3.1.1
+executing==2.2.0
+contexttimer==0.3.3
+lazy_loader==0.4
+wrapt==1.17.2
+webdataset==0.2.111
+blis==1.3.0
+idna==3.10
+timm==0.4.12
+einops==0.8.1
+packaging==24.2
+decorator==5.2.1
+filelock==3.18.0
+python-slugify==8.0.4
+cycler==0.12.1
+charset-normalizer==3.4.2
+pydantic==2.11.5
+pydeck==0.9.1
+tzdata==2025.2
+jedi==0.19.2
+aiohappyeyeballs==2.6.1
+nvidia-nvjitlink-cu12==12.4.127
+salesforce-lavis==1.0.2
+parso==0.8.4
+nvidia-nccl-cu12==2.21.5
+toml==0.10.2
+python-dateutil==2.9.0.post0
+rich==14.0.0
+tqdm==4.67.1
+rpds-py==0.25.1
+opencv-python-headless==4.5.5.64
+tornado==6.5.1
+propcache==0.3.1
+webencodings==0.5.1
+murmurhash==1.0.13
+contourpy==1.3.2
+joblib==1.5.1
+networkx==3.4.2
+six==1.17.0
+markdown-it-py==3.0.0
+nvidia-cuda-cupti-cu12==12.4.127
+msgpack==1.1.0
+sentencepiece==0.2.0
+cymem==2.0.11
+nvidia-cufft-cu12==11.2.1.3
+absl-py==2.2.2
+hjson==3.1.0
+mpmath==1.3.0
+pydantic_core==2.33.2
+psutil==7.0.0
+nvidia-ml-py==12.575.51
+pyarrow==20.0.0
+kiwisolver==1.4.8
+sympy==1.13.1
+ninja==1.11.1.4
+rouge_score==0.1.2
+deepspeed==0.16.10+b666844f
+spacy-legacy==3.0.12
+pycocoevalcap==1.2
+pexpect==4.9.0
+ftfy==6.3.1
+protobuf==6.31.0
+urllib3==2.4.0
+wheel==0.45.1
+nltk==3.9.1
+streamlit==1.45.1
+wasabi==1.1.3
+pre_commit==4.2.0
+safetensors==0.5.3
+jsonschema-specifications==2025.4.1
+langcodes==3.5.0
+GitPython==3.1.44
+blinker==1.9.0
+torchvision==0.21.0
+MarkupSafe==3.0.2
+dill==0.3.8
+yacs==0.1.8
+pathlib==1.0.1
+scikit-learn==1.6.1
+cffi==1.17.1
+pycparser==2.22
+flash-attn==2.7.1.post1
+cryptography==45.0.3
+pycryptodome==3.23.0
+cheroot==10.0.1
+more-itertools==10.7.0
+setproctitle==1.3.6
+delta-center-client==0.0.4
+jmespath==0.10.0
+xxhash==3.5.0
+pip==25.1.1
+aliyun-python-sdk-core==2.16.0
+jaraco.functools==4.1.0
+bigmodelvis==0.0.1
+aiohttp==3.12.2
+multiprocess==0.70.16
+opendelta==0.3.2
+docker-pycreds==0.4.0
+threadpoolctl==3.6.0
+click==8.2.1
+oss2==2.15.0
+crcmod==1.7
+transformers==4.52.3
+datasets==3.6.0
+jsonschema==4.24.0
+opencv-python==4.11.0.86
+wandb==0.19.11
+fsspec==2025.3.0
+tokenizers==0.21.1
+sentry-sdk==2.29.1
+preshed==3.0.10
+aliyun-python-sdk-kms==2.16.5
+huggingface-hub==0.32.1
+typer==0.16.0
+narwhals==1.41.0
+web.py==0.62
+autocommand==2.2.2
+importlib_metadata==8.0.0
+zipp==3.19.2
+jaraco.context==5.3.0
+typeguard==4.3.0
+jaraco.collections==5.1.0
+typing_extensions==4.12.2
+backports.tarfile==1.2.0
+jaraco.functools==4.0.1
+more-itertools==10.3.0
+platformdirs==4.2.2
+packaging==24.2
+tomli==2.0.1
+jaraco.text==3.12.1
+wheel==0.45.1
+inflect==7.3.1

ProtT3/all_checkpoints/stage2_07021249/wandb/run-20250702_130101-ttc1macp/files/wandb-metadata.json

@@ -0,0 +1,104 @@
+{
+  "os": "Linux-5.10.134-008.16.kangaroo.al8.x86_64-x86_64-with-glibc2.35",
+  "python": "CPython 3.10.0",
+  "startedAt": "2025-07-02T05:01:02.468047Z",
+  "args": [
+    "--devices",
+    "0,1,2,3,4,5,6,7",
+    "--mode",
+    "train",
+    "--filename",
+    "stage2_07021249",
+    "--num_query_token",
+    "8",
+    "--save_every_n_epochs",
+    "5",
+    "--max_epochs",
+    "10",
+    "--batch_size",
+    "32",
+    "--precision",
+    "bf16-mixed",
+    "--num_workers",
+    "8",
+    "--plm_model",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m",
+    "--bert_name",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft",
+    "--llm_name",
+    "/oss/wangyujia/BIO/pretrain_output/qwen2.5-7b-instruct-bio/bio_all/save1epoch/checkpoint-1300",
+    "--llm_tune",
+    "mid_lora",
+    "--mix_dataset",
+    "--stage1_path",
+    "/nas/shared/kilab/wangyujia/ProtT3/all_checkpoints/stage1_06290009_deepspeed/epoch=19.ckpt/converted.ckpt",
+    "--use_wandb_logger"
+  ],
+  "program": "/nas/shared/kilab/wangyujia/ProtT3/stage2.py",
+  "codePath": "stage2.py",
+  "email": "gia0603yucca@gmail.com",
+  "root": "./all_checkpoints/stage2_07021249/",
+  "host": "dsw-265304-6d4dbc55d6-l29sk",
+  "executable": "/root/miniconda3/envs/protT3/bin/python",
+  "codePathLocal": "stage2.py",
+  "cpu_count": 64,
+  "cpu_count_logical": 64,
+  "gpu": "NVIDIA A800-SXM4-80GB",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1623302262784",
+      "used": "987885568"
+    }
+  },
+  "memory": {
+    "total": "549755813888"
+  },
+  "cpu": {
+    "count": 64,
+    "countLogical": 64
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    }
+  ],
+  "cudaVersion": "12.1"
+}

ProtT3/all_checkpoints/stage2_07021249/wandb/run-20250702_130101-ttc1macp/files/wandb-summary.json

@@ -0,0 +1 @@
+{"dataset0/bleu4":11.860699653625488,"dataset0/rouge_2":13.413575172424316,"trainer/global_step":16819,"dataloader2/val loss/dataloader_idx_2":5.575827121734619,"_wandb":{"runtime":16587},"dataloader0/val loss/dataloader_idx_0":2.2659671306610107,"_timestamp":1.7514490380975404e+09,"_runtime":16575.629722537,"_step":345,"dataset0/bleu2":15.341177940368652,"loss":1.30044424533844,"dataset0/meteor_score":26.009353637695312,"dataset0/rouge_1":21.952245712280273,"epoch":9,"lr":1.2202456673549023e-05,"dataset0/acc":0.009999999776482582,"dataset0/rouge_l":19.765010833740234}

ProtT3/all_checkpoints/stage2_07021249/wandb/run-20250702_130101-ttc1macp/logs/debug-internal.log

@@ -0,0 +1,17 @@
+{"time":"2025-07-02T13:01:02.517398459+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.11","symlink path":"all_checkpoints/stage2_07021249/wandb/run-20250702_130101-ttc1macp/logs/debug-core.log"}
+{"time":"2025-07-02T13:01:03.639276653+08:00","level":"INFO","msg":"created new stream","id":"ttc1macp"}
+{"time":"2025-07-02T13:01:03.63932174+08:00","level":"INFO","msg":"stream: started","id":"ttc1macp"}
+{"time":"2025-07-02T13:01:03.639331986+08:00","level":"INFO","msg":"handler: started","stream_id":"ttc1macp"}
+{"time":"2025-07-02T13:01:03.639357726+08:00","level":"INFO","msg":"sender: started","stream_id":"ttc1macp"}
+{"time":"2025-07-02T13:01:03.639363515+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"ttc1macp"}
+{"time":"2025-07-02T13:01:05.044613784+08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2025-07-02T14:25:40.366902458+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07021249/ttc1macp/file_stream\": unexpected EOF"}
+{"time":"2025-07-02T15:45:50.370249421+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-07-02T17:37:30.121166513+08:00","level":"INFO","msg":"stream: closing","id":"ttc1macp"}
+{"time":"2025-07-02T17:37:30.121209819+08:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2025-07-02T17:37:30.137879266+08:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2025-07-02T17:39:43.997948982+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-07-02T17:39:45.216379261+08:00","level":"INFO","msg":"handler: closed","stream_id":"ttc1macp"}
+{"time":"2025-07-02T17:39:45.216437206+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"ttc1macp"}
+{"time":"2025-07-02T17:39:45.216450748+08:00","level":"INFO","msg":"sender: closed","stream_id":"ttc1macp"}
+{"time":"2025-07-02T17:39:45.269415307+08:00","level":"INFO","msg":"stream: closed","id":"ttc1macp"}

ProtT3/all_checkpoints/stage2_07021249/wandb/run-20250702_130101-ttc1macp/logs/debug.log

@@ -0,0 +1,24 @@
+2025-07-02 13:01:02,446 INFO    MainThread:2341 [wandb_setup.py:_flush():70] Current SDK version is 0.19.11
+2025-07-02 13:01:02,446 INFO    MainThread:2341 [wandb_setup.py:_flush():70] Configure stats pid to 2341
+2025-07-02 13:01:02,446 INFO    MainThread:2341 [wandb_setup.py:_flush():70] Loading settings from /root/.config/wandb/settings
+2025-07-02 13:01:02,446 INFO    MainThread:2341 [wandb_setup.py:_flush():70] Loading settings from /nas/shared/kilab/wangyujia/ProtT3/wandb/settings
+2025-07-02 13:01:02,446 INFO    MainThread:2341 [wandb_setup.py:_flush():70] Loading settings from environment variables
+2025-07-02 13:01:02,446 INFO    MainThread:2341 [wandb_init.py:setup_run_log_directory():724] Logging user logs to ./all_checkpoints/stage2_07021249/wandb/run-20250702_130101-ttc1macp/logs/debug.log
+2025-07-02 13:01:02,446 INFO    MainThread:2341 [wandb_init.py:setup_run_log_directory():725] Logging internal logs to ./all_checkpoints/stage2_07021249/wandb/run-20250702_130101-ttc1macp/logs/debug-internal.log
+2025-07-02 13:01:02,446 INFO    MainThread:2341 [wandb_init.py:init():852] calling init triggers
+2025-07-02 13:01:02,446 INFO    MainThread:2341 [wandb_init.py:init():857] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-07-02 13:01:02,446 INFO    MainThread:2341 [wandb_init.py:init():893] starting backend
+2025-07-02 13:01:02,446 INFO    MainThread:2341 [wandb_init.py:init():897] sending inform_init request
+2025-07-02 13:01:02,449 INFO    MainThread:2341 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-07-02 13:01:02,449 INFO    MainThread:2341 [wandb_init.py:init():907] backend started and connected
+2025-07-02 13:01:02,468 INFO    MainThread:2341 [wandb_init.py:init():1005] updated telemetry
+2025-07-02 13:01:02,498 INFO    MainThread:2341 [wandb_init.py:init():1029] communicating run to backend with 90.0 second timeout
+2025-07-02 13:01:04,915 INFO    MainThread:2341 [wandb_init.py:init():1104] starting run threads in backend
+2025-07-02 13:01:05,214 INFO    MainThread:2341 [wandb_run.py:_console_start():2573] atexit reg
+2025-07-02 13:01:05,215 INFO    MainThread:2341 [wandb_run.py:_redirect():2421] redirect: wrap_raw
+2025-07-02 13:01:05,215 INFO    MainThread:2341 [wandb_run.py:_redirect():2490] Wrapping output streams.
+2025-07-02 13:01:05,215 INFO    MainThread:2341 [wandb_run.py:_redirect():2513] Redirects installed.
+2025-07-02 13:01:05,218 INFO    MainThread:2341 [wandb_init.py:init():1150] run started, returning control to user process
+2025-07-02 13:01:12,114 INFO    MainThread:2341 [wandb_run.py:_config_callback():1436] config_cb None None {'filename': 'stage2_07021249', 'seed': 42, 'mode': 'train', 'strategy': 'deepspeed', 'accelerator': 'gpu', 'devices': '0,1,2,3,4,5,6,7', 'precision': 'bf16-mixed', 'max_epochs': 10, 'accumulate_grad_batches': 1, 'check_val_every_n_epoch': 1, 'enable_flash': False, 'use_wandb_logger': True, 'mix_dataset': True, 'save_every_n_epochs': 5, 'bert_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft', 'cross_attention_freq': 2, 'num_query_token': 8, 'llm_name': '/oss/wangyujia/BIO/pretrain_output/qwen2.5-7b-instruct-bio/bio_all/save1epoch/checkpoint-1300', 'num_beams': 5, 'do_sample': False, 'max_inference_len': 128, 'min_inference_len': 1, 'llm_tune': 'mid_lora', 'peft_config': '', 'peft_dir': '', 'plm_model': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m', 'plm_tune': 'freeze', 'lora_r': 8, 'lora_alpha': 16, 'lora_dropout': 0.1, 'enbale_gradient_checkpointing': False, 'weight_decay': 0.05, 'init_lr': 0.0001, 'min_lr': 1e-05, 'warmup_lr': 1e-06, 'warmup_steps': 1000, 'lr_decay_rate': 0.9, 'scheduler': 'linear_warmup_cosine_lr', 'stage1_path': '/nas/shared/kilab/wangyujia/ProtT3/all_checkpoints/stage1_06290009_deepspeed/epoch=19.ckpt/converted.ckpt', 'stage2_path': '', 'init_checkpoint': '', 'caption_eval_epoch': 10, 'num_workers': 8, 'batch_size': 32, 'inference_batch_size': 4, 'root': 'data', 'text_max_len': 128, 'q_max_len': 29, 'a_max_len': 36, 'prot_max_len': 1024, 'prompt': 'The protein has the following properties: ', 'filter_side_qa': False}
+2025-07-02 17:37:30,072 INFO    MsgRouterThr:2341 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles.

ProtT3/all_checkpoints/stage2_07041521/wandb/debug-internal.log

@@ -0,0 +1,95 @@
+{"time":"2025-07-04T15:46:08.966654664+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.11","symlink path":"all_checkpoints/stage2_07041521/wandb/run-20250704_154608-ds7lrt6r/logs/debug-core.log"}
+{"time":"2025-07-04T15:46:10.62729617+08:00","level":"INFO","msg":"created new stream","id":"ds7lrt6r"}
+{"time":"2025-07-04T15:46:10.627339189+08:00","level":"INFO","msg":"stream: started","id":"ds7lrt6r"}
+{"time":"2025-07-04T15:46:10.627374947+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"ds7lrt6r"}
+{"time":"2025-07-04T15:46:10.627404904+08:00","level":"INFO","msg":"handler: started","stream_id":"ds7lrt6r"}
+{"time":"2025-07-04T15:46:10.627398441+08:00","level":"INFO","msg":"sender: started","stream_id":"ds7lrt6r"}
+{"time":"2025-07-04T15:46:12.482815718+08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2025-07-04T21:58:22.739226816+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": read tcp 10.1.8.17:36512->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-04T22:01:34.73927758+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": read tcp 10.1.8.17:56674->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-04T22:01:43.328655381+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-07-04T22:02:15.686026868+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-07-04T22:02:50.214808897+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-07-04T22:03:28.950188225+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-07-04T22:04:14.982810813+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-07-04T22:05:22.734102314+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-07-04T22:06:52.750047504+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-07-04T22:08:22.775414283+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-07-04T22:08:43.307492899+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": unexpected EOF"}
+{"time":"2025-07-04T22:09:52.781192276+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-07-04T22:11:13.328660156+08:00","level":"WARN","msg":"sender: taking a long time","seconds":600.000386663,"work":"WorkRecord(*service_go_proto.Request_StopStatus); Control(local:true mailbox_slot:\"2nedcn0bl5yp\" connection_id:\"127.0.0.1:57318\")"}
+{"time":"2025-07-04T22:11:22.78819562+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-07-04T22:12:52.789417986+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-07-04T22:13:05.234783312+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":600.000850585,"work":"WorkRecord(*service_go_proto.Record_OutputRaw); Control(connection_id:\"127.0.0.1:57318\")"}
+{"time":"2025-07-04T22:13:12.48514539+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":600.000329899,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2025-07-04T22:13:12.516285255+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":600.00042574,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2025-07-04T22:14:22.815202117+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-07-04T22:15:52.82570124+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-07-04T22:17:22.890129793+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-07-04T22:18:29.532924447+08:00","level":"INFO","msg":"sender: succeeded after taking longer than expected","seconds":1036.204664967,"work":"WorkRecord(*service_go_proto.Request_StopStatus); Control(local:true mailbox_slot:\"2nedcn0bl5yp\" connection_id:\"127.0.0.1:57318\")"}
+{"time":"2025-07-04T22:18:29.532969506+08:00","level":"INFO","msg":"runwork: succeeded after taking longer than expected","seconds":924.299032896,"work":"WorkRecord(*service_go_proto.Record_OutputRaw); Control(connection_id:\"127.0.0.1:57318\")"}
+{"time":"2025-07-04T22:18:29.532991216+08:00","level":"INFO","msg":"runwork: succeeded after taking longer than expected","seconds":917.048223115,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2025-07-04T22:18:29.532997449+08:00","level":"INFO","msg":"runwork: succeeded after taking longer than expected","seconds":917.017162756,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2025-07-04T22:22:02.242147225+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": read tcp 10.1.8.17:53384->104.21.20.172:443: read: connection reset by peer"}
+{"time":"2025-07-04T22:26:53.14669432+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": unexpected EOF"}
+{"time":"2025-07-04T22:27:48.780316277+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": unexpected EOF"}
+{"time":"2025-07-04T22:31:07.795211328+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": read tcp 10.1.8.17:59586->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-07-04T22:34:24.403211244+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": read tcp 10.1.8.17:36792->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-04T22:42:22.611231819+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": read tcp 10.1.8.17:52098->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-04T22:43:30.389313147+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": read tcp 10.1.8.17:48310->104.21.20.172:443: read: connection reset by peer"}
+{"time":"2025-07-04T22:44:01.015059936+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": unexpected EOF"}
+{"time":"2025-07-04T22:46:14.846692259+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": unexpected EOF"}
+{"time":"2025-07-04T22:48:31.349968366+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": read tcp 10.1.8.17:39256->104.21.20.172:443: read: connection reset by peer"}
+{"time":"2025-07-04T22:49:38.015743829+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": read tcp 10.1.8.17:38292->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-07-04T22:50:09.683679573+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": http2: client conn is closed"}
+{"time":"2025-07-04T22:52:08.558045187+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": read tcp 10.1.8.17:59988->104.21.20.172:443: read: connection reset by peer"}
+{"time":"2025-07-04T22:54:22.824301514+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": read tcp 10.1.8.17:38894->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-07-04T22:59:26.307679579+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": read tcp 10.1.8.17:51684->104.21.20.172:443: read: connection reset by peer"}
+{"time":"2025-07-04T23:02:32.979198883+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": read tcp 10.1.8.17:58210->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-04T23:10:21.459215862+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": read tcp 10.1.8.17:35172->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-04T23:11:15.167490198+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": read tcp 10.1.8.17:54942->104.21.20.172:443: read: connection reset by peer"}
+{"time":"2025-07-04T23:16:10.863292487+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": read tcp 10.1.8.17:43428->104.21.20.172:443: read: connection reset by peer"}
+{"time":"2025-07-04T23:20:22.264503678+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2025-07-04T23:23:20.723212059+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": read tcp 10.1.8.17:38048->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-07-04T23:25:26.885991394+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": read tcp 10.1.8.17:42070->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-07-04T23:26:43.632088162+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-07-04T23:29:40.115213912+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": read tcp 10.1.8.17:39740->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-07-04T23:30:43.633929839+08:00","level":"ERROR","msg":"sender: sendStopStatus: failed to get run stopped status: net/http: request canceled (Client.Timeout or context cancellation while reading body)"}
+{"time":"2025-07-04T23:31:28.634946467+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-07-04T23:32:54.67521322+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": read tcp 10.1.8.17:51016->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-04T23:39:42.739210995+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": read tcp 10.1.8.17:58242->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-07-04T23:45:14.003206141+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": read tcp 10.1.8.17:41730->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-04T23:48:41.876197491+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": read tcp 10.1.8.17:39056->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-04T23:52:10.25916891+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": read tcp 10.1.8.17:56968->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-04T23:55:53.491192209+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": read tcp 10.1.8.17:52384->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-04T23:58:33.235181122+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": read tcp 10.1.8.17:41192->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-05T00:01:39.09118175+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": read tcp 10.1.8.17:54870->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-05T00:03:30.660362141+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": unexpected EOF"}
+{"time":"2025-07-05T00:06:22.739197171+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": read tcp 10.1.8.17:56300->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-07-05T00:07:01.140306954+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": read tcp 10.1.8.17:33830->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-07-05T00:10:11.603177492+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": read tcp 10.1.8.17:38990->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-07-05T00:17:09.907203145+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": read tcp 10.1.8.17:58654->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-07-05T00:20:10.131225125+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": read tcp 10.1.8.17:39570->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-05T00:23:22.643197817+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": read tcp 10.1.8.17:57940->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-05T00:26:13.651201419+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": read tcp 10.1.8.17:37792->172.67.193.61:443: read: connection timed out"}
+{"time":"2025-07-05T00:28:56.467221564+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": read tcp 10.1.8.17:38640->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-07-05T00:32:27.924195852+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": read tcp 10.1.8.17:33266->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-07-05T00:33:00.356828932+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": read tcp 10.1.8.17:56794->104.21.20.172:443: read: connection reset by peer"}
+{"time":"2025-07-05T00:36:52.115188168+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": read tcp 10.1.8.17:44060->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-07-05T00:40:28.83076072+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/graphql\": context deadline exceeded"}
+{"time":"2025-07-05T00:41:16.304855216+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": unexpected EOF"}
+{"time":"2025-07-05T00:44:59.539204741+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": read tcp 10.1.8.17:54050->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-07-05T00:46:47.38055068+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": read tcp 10.1.8.17:47426->172.67.193.61:443: read: connection reset by peer"}
+{"time":"2025-07-05T00:48:09.874069624+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": read tcp 10.1.8.17:60378->104.21.20.172:443: read: connection reset by peer"}
+{"time":"2025-07-05T00:50:24.818784704+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": unexpected EOF"}
+{"time":"2025-07-05T01:02:13.78019116+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": read tcp 10.1.8.17:44352->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-07-05T01:10:57.377024443+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": context deadline exceeded"}
+{"time":"2025-07-05T01:20:40.211190451+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.bandw.top/files/gia0603yucca/stage2_07041521/ds7lrt6r/file_stream\": read tcp 10.1.8.17:50414->104.21.20.172:443: read: connection timed out"}
+{"time":"2025-07-05T01:21:51.097875981+08:00","level":"INFO","msg":"stream: closing","id":"ds7lrt6r"}
+{"time":"2025-07-05T01:21:51.097937445+08:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2025-07-05T01:21:51.099273597+08:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2025-07-05T01:21:58.380081154+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-07-05T01:22:06.296069471+08:00","level":"INFO","msg":"handler: closed","stream_id":"ds7lrt6r"}
+{"time":"2025-07-05T01:22:06.296102451+08:00","level":"INFO","msg":"sender: closed","stream_id":"ds7lrt6r"}
+{"time":"2025-07-05T01:22:06.296100202+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"ds7lrt6r"}
+{"time":"2025-07-05T01:22:06.302257653+08:00","level":"INFO","msg":"stream: closed","id":"ds7lrt6r"}

ProtT3/all_checkpoints/stage2_07041521/wandb/debug.log

@@ -0,0 +1,24 @@
+2025-07-04 15:46:08,903 INFO    MainThread:56865 [wandb_setup.py:_flush():70] Current SDK version is 0.19.11
+2025-07-04 15:46:08,903 INFO    MainThread:56865 [wandb_setup.py:_flush():70] Configure stats pid to 56865
+2025-07-04 15:46:08,903 INFO    MainThread:56865 [wandb_setup.py:_flush():70] Loading settings from /root/.config/wandb/settings
+2025-07-04 15:46:08,903 INFO    MainThread:56865 [wandb_setup.py:_flush():70] Loading settings from /nas/shared/kilab/wangyujia/ProtT3/wandb/settings
+2025-07-04 15:46:08,903 INFO    MainThread:56865 [wandb_setup.py:_flush():70] Loading settings from environment variables
+2025-07-04 15:46:08,903 INFO    MainThread:56865 [wandb_init.py:setup_run_log_directory():724] Logging user logs to ./all_checkpoints/stage2_07041521/wandb/run-20250704_154608-ds7lrt6r/logs/debug.log
+2025-07-04 15:46:08,903 INFO    MainThread:56865 [wandb_init.py:setup_run_log_directory():725] Logging internal logs to ./all_checkpoints/stage2_07041521/wandb/run-20250704_154608-ds7lrt6r/logs/debug-internal.log
+2025-07-04 15:46:08,903 INFO    MainThread:56865 [wandb_init.py:init():852] calling init triggers
+2025-07-04 15:46:08,903 INFO    MainThread:56865 [wandb_init.py:init():857] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-07-04 15:46:08,903 INFO    MainThread:56865 [wandb_init.py:init():893] starting backend
+2025-07-04 15:46:08,903 INFO    MainThread:56865 [wandb_init.py:init():897] sending inform_init request
+2025-07-04 15:46:08,914 INFO    MainThread:56865 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-07-04 15:46:08,916 INFO    MainThread:56865 [wandb_init.py:init():907] backend started and connected
+2025-07-04 15:46:08,917 INFO    MainThread:56865 [wandb_init.py:init():1005] updated telemetry
+2025-07-04 15:46:08,922 INFO    MainThread:56865 [wandb_init.py:init():1029] communicating run to backend with 90.0 second timeout
+2025-07-04 15:46:12,402 INFO    MainThread:56865 [wandb_init.py:init():1104] starting run threads in backend
+2025-07-04 15:46:12,679 INFO    MainThread:56865 [wandb_run.py:_console_start():2573] atexit reg
+2025-07-04 15:46:12,680 INFO    MainThread:56865 [wandb_run.py:_redirect():2421] redirect: wrap_raw
+2025-07-04 15:46:12,685 INFO    MainThread:56865 [wandb_run.py:_redirect():2490] Wrapping output streams.
+2025-07-04 15:46:12,686 INFO    MainThread:56865 [wandb_run.py:_redirect():2513] Redirects installed.
+2025-07-04 15:46:12,697 INFO    MainThread:56865 [wandb_init.py:init():1150] run started, returning control to user process
+2025-07-04 15:46:21,744 INFO    MainThread:56865 [wandb_run.py:_config_callback():1436] config_cb None None {'filename': 'stage2_07041521', 'seed': 42, 'mode': 'train', 'strategy': 'deepspeed', 'accelerator': 'gpu', 'devices': '0,1,2,3,4,5,6,7', 'precision': 'bf16-mixed', 'max_epochs': 15, 'accumulate_grad_batches': 1, 'check_val_every_n_epoch': 1, 'enable_flash': False, 'use_wandb_logger': True, 'mix_dataset': True, 'save_every_n_epochs': 5, 'bert_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft', 'cross_attention_freq': 2, 'num_query_token': 8, 'llm_name': '/oss/wangyujia/BIO/pretrain_output/qwen2.5-7b-instruct-bio/bio_all/save1epoch/checkpoint-1300', 'num_beams': 5, 'do_sample': False, 'max_inference_len': 128, 'min_inference_len': 1, 'llm_tune': 'mid_lora', 'peft_config': '', 'peft_dir': '', 'plm_model': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m', 'plm_tune': 'freeze', 'lora_r': 8, 'lora_alpha': 16, 'lora_dropout': 0.1, 'enbale_gradient_checkpointing': False, 'weight_decay': 0.05, 'init_lr': 0.0001, 'min_lr': 1e-05, 'warmup_lr': 1e-06, 'warmup_steps': 1000, 'lr_decay_rate': 0.9, 'scheduler': 'linear_warmup_cosine_lr', 'stage1_path': '/nas/shared/kilab/wangyujia/ProtT3/all_checkpoints/stage1_06290009_deepspeed/epoch=19.ckpt/converted.ckpt', 'stage2_path': '', 'init_checkpoint': '', 'caption_eval_epoch': 10, 'num_workers': 8, 'batch_size': 32, 'inference_batch_size': 4, 'root': 'data', 'text_max_len': 128, 'q_max_len': 29, 'a_max_len': 36, 'prot_max_len': 1024, 'prompt': 'The protein has the following properties: ', 'filter_side_qa': False}
+2025-07-05 01:21:51,095 INFO    MsgRouterThr:56865 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles.

ProtT3/all_checkpoints/stage2_07041521/wandb/run-20250704_152253-wgyb9m42/files/config.yaml

@@ -0,0 +1,150 @@
+_wandb:
+    value:
+        cli_version: 0.19.11
+        m:
+            - "1": trainer/global_step
+              "6":
+                - 3
+              "7": []
+        python_version: 3.10.0
+        t:
+            "1":
+                - 1
+                - 5
+                - 9
+                - 11
+                - 33
+                - 41
+                - 49
+                - 53
+                - 55
+                - 63
+                - 103
+            "2":
+                - 1
+                - 5
+                - 9
+                - 11
+                - 33
+                - 41
+                - 49
+                - 53
+                - 55
+                - 63
+                - 103
+            "3":
+                - 7
+                - 23
+                - 55
+                - 66
+            "4": 3.10.0
+            "5": 0.19.11
+            "6": 4.52.3
+            "8":
+                - 5
+            "12": 0.19.11
+            "13": linux-x86_64
+a_max_len:
+    value: 36
+accelerator:
+    value: gpu
+accumulate_grad_batches:
+    value: 1
+batch_size:
+    value: 32
+bert_name:
+    value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft
+caption_eval_epoch:
+    value: 10
+check_val_every_n_epoch:
+    value: 1
+cross_attention_freq:
+    value: 2
+devices:
+    value: 0,1,2,3,4,5,6,7
+do_sample:
+    value: false
+enable_flash:
+    value: false
+enbale_gradient_checkpointing:
+    value: false
+filename:
+    value: stage2_07041521
+filter_side_qa:
+    value: false
+inference_batch_size:
+    value: 4
+init_checkpoint:
+    value: ""
+init_lr:
+    value: 0.0001
+llm_name:
+    value: /oss/wangyujia/BIO/pretrain_output/qwen2.5-7b-instruct-bio/bio_all/save1epoch/checkpoint-1300
+llm_tune:
+    value: mid_lora
+lora_alpha:
+    value: 16
+lora_dropout:
+    value: 0.1
+lora_r:
+    value: 8
+lr_decay_rate:
+    value: 0.9
+max_epochs:
+    value: 15
+max_inference_len:
+    value: 128
+min_inference_len:
+    value: 1
+min_lr:
+    value: 1e-05
+mix_dataset:
+    value: true
+mode:
+    value: train
+num_beams:
+    value: 5
+num_query_token:
+    value: 8
+num_workers:
+    value: 8
+peft_config:
+    value: ""
+peft_dir:
+    value: ""
+plm_model:
+    value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m
+plm_tune:
+    value: freeze
+precision:
+    value: bf16-mixed
+prompt:
+    value: 'The protein has the following properties: '
+prot_max_len:
+    value: 1024
+q_max_len:
+    value: 29
+root:
+    value: data
+save_every_n_epochs:
+    value: 5
+scheduler:
+    value: linear_warmup_cosine_lr
+seed:
+    value: 42
+stage1_path:
+    value: /nas/shared/kilab/wangyujia/ProtT3/all_checkpoints/stage1_06290009_deepspeed/epoch=19.ckpt/converted.ckpt
+stage2_path:
+    value: ""
+strategy:
+    value: deepspeed
+text_max_len:
+    value: 128
+use_wandb_logger:
+    value: true
+warmup_lr:
+    value: 1e-06
+warmup_steps:
+    value: 1000
+weight_decay:
+    value: 0.05

ProtT3/all_checkpoints/stage2_07041521/wandb/run-20250704_152253-wgyb9m42/files/output.log

@@ -0,0 +1,131 @@
+/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:654: Checkpoint directory /nas/shared/kilab/wangyujia/ProtT3/all_checkpoints/stage2_07041521 exists and is not empty.
+Enabling DeepSpeed BF16. Model parameters and inputs will be cast to `bfloat16`.
+LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]
+
+  | Name  | Type     | Params | Mode
+-------------------------------------------
+0 | blip2 | Blip2OPT | 7.9 B  | train
+-------------------------------------------
+104 M     Trainable params
+7.8 B     Non-trainable params
+7.9 B     Total params
+31,459.025Total estimated model params size (MB)
+174       Modules in train mode
+1203      Modules in eval mode
+Sanity Checking DataLoader 0:   0%|                                               | 0/2 [00:00<?, ?it/s]
+Traceback (most recent call last):
+  File "/nas/shared/kilab/wangyujia/ProtT3/stage2.py", line 130, in <module>
+    main(get_args())
+  File "/nas/shared/kilab/wangyujia/ProtT3/stage2.py", line 92, in main
+    trainer.fit(model, datamodule=dm)#, ckpt_path=args.ckpt_path)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 561, in fit
+    call._call_and_handle_interrupt(
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 47, in _call_and_handle_interrupt
+    return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 105, in launch
+    return function(*args, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 599, in _fit_impl
+    self._run(model, ckpt_path=ckpt_path)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1012, in _run
+    results = self._run_stage()
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1054, in _run_stage
+    self._run_sanity_check()
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1083, in _run_sanity_check
+    val_loop.run()
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/loops/utilities.py", line 179, in _decorator
+    return loop_run(self, *args, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/loops/evaluation_loop.py", line 145, in run
+    self._evaluation_step(batch, batch_idx, dataloader_idx, dataloader_iter)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/loops/evaluation_loop.py", line 437, in _evaluation_step
+    output = call._call_strategy_hook(trainer, hook_name, *step_args)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 328, in _call_strategy_hook
+    output = fn(*args, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 411, in validation_step
+    return self._forward_redirection(self.model, self.lightning_module, "validation_step", *args, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 641, in __call__
+    wrapper_output = wrapper_module(*args, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/deepspeed/utils/nvtx.py", line 20, in wrapped_fn
+    ret_val = func(*args, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/deepspeed/runtime/engine.py", line 2054, in forward
+    loss = self.module(*inputs, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1845, in _call_impl
+    return inner()
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1793, in inner
+    result = forward_call(*args, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 634, in wrapped_forward
+    out = method(*_args, **_kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context
+    return func(*args, **kwargs)
+  File "/nas/shared/kilab/wangyujia/ProtT3/model/blip2_stage2.py", line 119, in validation_step
+    loss = self.blip2(batch)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/nas/shared/kilab/wangyujia/ProtT3/model/blip2_opt.py", line 222, in forward
+    prot_batch, prompt_batch, text_batch = batch
+ValueError: not enough values to unpack (expected 3, got 2)
+[rank0]: Traceback (most recent call last):
+[rank0]:   File "/nas/shared/kilab/wangyujia/ProtT3/stage2.py", line 130, in <module>
+[rank0]:     main(get_args())
+[rank0]:   File "/nas/shared/kilab/wangyujia/ProtT3/stage2.py", line 92, in main
+[rank0]:     trainer.fit(model, datamodule=dm)#, ckpt_path=args.ckpt_path)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 561, in fit
+[rank0]:     call._call_and_handle_interrupt(
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 47, in _call_and_handle_interrupt
+[rank0]:     return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 105, in launch
+[rank0]:     return function(*args, **kwargs)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 599, in _fit_impl
+[rank0]:     self._run(model, ckpt_path=ckpt_path)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1012, in _run
+[rank0]:     results = self._run_stage()
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1054, in _run_stage
+[rank0]:     self._run_sanity_check()
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1083, in _run_sanity_check
+[rank0]:     val_loop.run()
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/loops/utilities.py", line 179, in _decorator
+[rank0]:     return loop_run(self, *args, **kwargs)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/loops/evaluation_loop.py", line 145, in run
+[rank0]:     self._evaluation_step(batch, batch_idx, dataloader_idx, dataloader_iter)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/loops/evaluation_loop.py", line 437, in _evaluation_step
+[rank0]:     output = call._call_strategy_hook(trainer, hook_name, *step_args)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 328, in _call_strategy_hook
+[rank0]:     output = fn(*args, **kwargs)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 411, in validation_step
+[rank0]:     return self._forward_redirection(self.model, self.lightning_module, "validation_step", *args, **kwargs)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 641, in __call__
+[rank0]:     wrapper_output = wrapper_module(*args, **kwargs)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
+[rank0]:     return self._call_impl(*args, **kwargs)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
+[rank0]:     return forward_call(*args, **kwargs)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/deepspeed/utils/nvtx.py", line 20, in wrapped_fn
+[rank0]:     ret_val = func(*args, **kwargs)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/deepspeed/runtime/engine.py", line 2054, in forward
+[rank0]:     loss = self.module(*inputs, **kwargs)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
+[rank0]:     return self._call_impl(*args, **kwargs)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1845, in _call_impl
+[rank0]:     return inner()
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1793, in inner
+[rank0]:     result = forward_call(*args, **kwargs)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 634, in wrapped_forward
+[rank0]:     out = method(*_args, **_kwargs)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context
+[rank0]:     return func(*args, **kwargs)
+[rank0]:   File "/nas/shared/kilab/wangyujia/ProtT3/model/blip2_stage2.py", line 119, in validation_step
+[rank0]:     loss = self.blip2(batch)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
+[rank0]:     return self._call_impl(*args, **kwargs)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
+[rank0]:     return forward_call(*args, **kwargs)
+[rank0]:   File "/nas/shared/kilab/wangyujia/ProtT3/model/blip2_opt.py", line 222, in forward
+[rank0]:     prot_batch, prompt_batch, text_batch = batch
+[rank0]: ValueError: not enough values to unpack (expected 3, got 2)

ProtT3/all_checkpoints/stage2_07041521/wandb/run-20250704_152253-wgyb9m42/files/requirements.txt

@@ -0,0 +1,225 @@
+pydantic_core==2.33.2
+psutil==7.0.0
+nvidia-cuda-nvrtc-cu12==12.4.127
+mpmath==1.3.0
+tzdata==2025.2
+contexttimer==0.3.3
+cycler==0.12.1
+python-magic==0.4.27
+pexpect==4.9.0
+sympy==1.13.1
+wrapt==1.17.2
+marisa-trie==1.2.1
+langcodes==3.5.0
+nvidia-nvtx-cu12==12.4.127
+ipython==8.36.0
+opencv-python-headless==4.5.5.64
+MarkupSafe==3.0.2
+jsonschema-specifications==2025.4.1
+wasabi==1.1.3
+blinker==1.9.0
+cfgv==3.4.0
+numpy==2.2.6
+idna==3.10
+nvidia-cufile-cu12==1.11.1.6
+ninja==1.11.1.4
+nvidia-nccl-cu12==2.21.5
+networkx==3.4.2
+certifi==2025.4.26
+deepspeed==0.16.10+b666844f
+pure_eval==0.2.3
+packaging==24.2
+nltk==3.9.1
+contourpy==1.3.2
+pre_commit==4.2.0
+nodeenv==1.9.1
+setuptools==78.1.1
+annotated-types==0.7.0
+multidict==6.4.4
+requests==2.32.3
+tornado==6.5.1
+triton==3.2.0
+pillow==11.2.1
+decord==0.6.0
+shellingham==1.5.4
+streamlit==1.45.1
+pydeck==0.9.1
+confection==0.1.5
+exceptiongroup==1.3.0
+prompt_toolkit==3.0.51
+text-unidecode==1.3
+nvidia-cufft-cu12==11.2.1.3
+antlr4-python3-runtime==4.9.3
+fairscale==0.4.4
+rouge_score==0.1.2
+nvidia-cudnn-cu12==9.1.0.70
+tqdm==4.67.1
+rich==14.0.0
+frozenlist==1.6.0
+webencodings==0.5.1
+altair==5.5.0
+opendatasets==0.1.22
+nvidia-curand-cu12==10.3.5.147
+protobuf==6.31.0
+asttokens==3.0.0
+wheel==0.45.1
+hf-xet==1.1.2
+weasel==0.4.1
+aiosignal==1.3.2
+absl-py==2.2.2
+thinc==8.3.6
+torchvision==0.21.0
+pandas==2.2.3
+fonttools==4.58.0
+bleach==6.2.0
+typing-inspection==0.4.1
+ftfy==6.3.1
+typing_extensions==4.13.2
+nvidia-ml-py==12.575.51
+python-slugify==8.0.4
+lightning-utilities==0.14.3
+py-cpuinfo==9.0.0
+smmap==5.0.2
+regex==2024.11.6
+scikit-image==0.25.2
+iopath==0.1.10
+spacy-legacy==3.0.12
+hjson==3.1.0
+executing==2.2.0
+kiwisolver==1.4.8
+scipy==1.15.3
+aiohappyeyeballs==2.6.1
+toml==0.10.2
+jedi==0.19.2
+GitPython==3.1.44
+ptyprocess==0.7.0
+kaggle==1.7.4.5
+braceexpand==0.1.7
+wcwidth==0.2.13
+nvidia-cuda-runtime-cu12==12.4.127
+pytorch-lightning==2.5.1.post0
+Jinja2==3.1.6
+urllib3==2.4.0
+watchdog==6.0.0
+filelock==3.18.0
+propcache==0.3.1
+torch==2.6.0
+nvidia-cusparse-cu12==12.3.1.170
+cymem==2.0.11
+nvidia-cusolver-cu12==11.6.1.9
+murmurhash==1.0.13
+catalogue==2.0.10
+yarl==1.20.0
+charset-normalizer==3.4.2
+gitdb==4.0.12
+matplotlib==3.10.3
+portalocker==3.1.1
+platformdirs==4.3.8
+async-timeout==5.0.1
+parso==0.8.4
+markdown-it-py==3.0.0
+omegaconf==2.3.0
+cloudpathlib==0.21.1
+nvidia-cusparselt-cu12==0.6.2
+spacy-loggers==1.0.5
+srsly==2.5.1
+identify==2.6.12
+rpds-py==0.25.1
+spacy==3.8.7
+matplotlib-inline==0.1.7
+smart-open==7.1.0
+pydantic==2.11.5
+mdurl==0.1.2
+virtualenv==20.31.2
+pytz==2025.2
+pycocotools==2.0.8
+six==1.17.0
+decorator==5.2.1
+referencing==0.36.2
+sentencepiece==0.2.0
+PyYAML==6.0.2
+pycocoevalcap==1.2
+imageio==2.37.0
+distlib==0.3.9
+pyarrow==20.0.0
+tenacity==9.1.2
+language_data==1.3.0
+nvidia-cuda-cupti-cu12==12.4.127
+blis==1.3.0
+Pygments==2.19.1
+tifffile==2025.5.10
+pyparsing==3.2.3
+cachetools==5.5.2
+safetensors==0.5.3
+attrs==25.3.0
+webdataset==0.2.111
+plotly==6.1.1
+nvidia-cublas-cu12==12.4.5.8
+timm==0.4.12
+torchmetrics==1.7.1
+nvidia-nvjitlink-cu12==12.4.127
+stack-data==0.6.3
+python-dateutil==2.9.0.post0
+lazy_loader==0.4
+traitlets==5.14.3
+einops==0.8.1
+salesforce-lavis==1.0.2
+joblib==1.5.1
+msgpack==1.1.0
+tokenizers==0.21.1
+sentry-sdk==2.29.1
+oss2==2.15.0
+setproctitle==1.3.6
+pip==25.1.1
+cffi==1.17.1
+transformers==4.52.3
+narwhals==1.41.0
+aliyun-python-sdk-core==2.16.0
+jsonschema==4.24.0
+flash-attn==2.7.1.post1
+preshed==3.0.10
+multiprocess==0.70.16
+cryptography==45.0.3
+aliyun-python-sdk-kms==2.16.5
+scikit-learn==1.6.1
+huggingface-hub==0.32.1
+crcmod==1.7
+typer==0.16.0
+web.py==0.62
+docker-pycreds==0.4.0
+xxhash==3.5.0
+bigmodelvis==0.0.1
+datasets==3.6.0
+more-itertools==10.7.0
+yacs==0.1.8
+jmespath==0.10.0
+aiohttp==3.12.2
+opencv-python==4.11.0.86
+pycparser==2.22
+threadpoolctl==3.6.0
+jaraco.functools==4.1.0
+click==8.2.1
+wandb==0.19.11
+opendelta==0.3.2
+pycryptodome==3.23.0
+pathlib==1.0.1
+dill==0.3.8
+fsspec==2025.3.0
+delta-center-client==0.0.4
+cheroot==10.0.1
+typing_extensions==4.12.2
+platformdirs==4.2.2
+jaraco.text==3.12.1
+packaging==24.2
+inflect==7.3.1
+jaraco.context==5.3.0
+wheel==0.45.1
+typeguard==4.3.0
+more-itertools==10.3.0
+tomli==2.0.1
+importlib_metadata==8.0.0
+backports.tarfile==1.2.0
+zipp==3.19.2
+jaraco.collections==5.1.0
+autocommand==2.2.2
+jaraco.functools==4.0.1

ProtT3/all_checkpoints/stage2_07041521/wandb/run-20250704_152253-wgyb9m42/files/wandb-metadata.json

@@ -0,0 +1,104 @@
+{
+  "os": "Linux-5.10.134-008.16.kangaroo.al8.x86_64-x86_64-with-glibc2.35",
+  "python": "CPython 3.10.0",
+  "startedAt": "2025-07-04T07:22:53.352152Z",
+  "args": [
+    "--devices",
+    "0,1,2,3,4,5,6,7",
+    "--mode",
+    "train",
+    "--filename",
+    "stage2_07041521",
+    "--num_query_token",
+    "8",
+    "--save_every_n_epochs",
+    "5",
+    "--max_epochs",
+    "15",
+    "--batch_size",
+    "32",
+    "--precision",
+    "bf16-mixed",
+    "--num_workers",
+    "8",
+    "--plm_model",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m",
+    "--bert_name",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft",
+    "--llm_name",
+    "/oss/wangyujia/BIO/pretrain_output/qwen2.5-7b-instruct-bio/bio_all/save1epoch/checkpoint-1300",
+    "--llm_tune",
+    "mid_lora",
+    "--mix_dataset",
+    "--stage1_path",
+    "/nas/shared/kilab/wangyujia/ProtT3/all_checkpoints/stage1_06290009_deepspeed/epoch=19.ckpt/converted.ckpt",
+    "--use_wandb_logger"
+  ],
+  "program": "/nas/shared/kilab/wangyujia/ProtT3/stage2.py",
+  "codePath": "stage2.py",
+  "email": "gia0603yucca@gmail.com",
+  "root": "./all_checkpoints/stage2_07041521/",
+  "host": "dsw-265304-b8d7644bb-bs7r7",
+  "executable": "/root/miniconda3/envs/protT3/bin/python",
+  "codePathLocal": "stage2.py",
+  "cpu_count": 64,
+  "cpu_count_logical": 64,
+  "gpu": "NVIDIA A800-SXM4-80GB",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1623302262784",
+      "used": "1266487296"
+    }
+  },
+  "memory": {
+    "total": "549755813888"
+  },
+  "cpu": {
+    "count": 64,
+    "countLogical": 64
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    }
+  ],
+  "cudaVersion": "12.1"
+}

ProtT3/all_checkpoints/stage2_07041521/wandb/run-20250704_152253-wgyb9m42/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":21}}

ProtT3/all_checkpoints/stage2_07041521/wandb/run-20250704_152253-wgyb9m42/logs/debug-internal.log

@@ -0,0 +1,15 @@
+{"time":"2025-07-04T15:22:53.388896985+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.11","symlink path":"all_checkpoints/stage2_07041521/wandb/run-20250704_152253-wgyb9m42/logs/debug-core.log"}
+{"time":"2025-07-04T15:22:56.842915105+08:00","level":"INFO","msg":"created new stream","id":"wgyb9m42"}
+{"time":"2025-07-04T15:22:56.861110866+08:00","level":"INFO","msg":"stream: started","id":"wgyb9m42"}
+{"time":"2025-07-04T15:22:56.861147295+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"wgyb9m42"}
+{"time":"2025-07-04T15:22:56.861334961+08:00","level":"INFO","msg":"sender: started","stream_id":"wgyb9m42"}
+{"time":"2025-07-04T15:22:56.86133724+08:00","level":"INFO","msg":"handler: started","stream_id":"wgyb9m42"}
+{"time":"2025-07-04T15:22:58.128508153+08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2025-07-04T15:23:14.819973413+08:00","level":"INFO","msg":"stream: closing","id":"wgyb9m42"}
+{"time":"2025-07-04T15:23:14.820051202+08:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2025-07-04T15:23:14.852874205+08:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2025-07-04T15:23:17.850226248+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-07-04T15:23:18.890479762+08:00","level":"INFO","msg":"handler: closed","stream_id":"wgyb9m42"}
+{"time":"2025-07-04T15:23:18.890514395+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"wgyb9m42"}
+{"time":"2025-07-04T15:23:18.890532968+08:00","level":"INFO","msg":"sender: closed","stream_id":"wgyb9m42"}
+{"time":"2025-07-04T15:23:18.901418972+08:00","level":"INFO","msg":"stream: closed","id":"wgyb9m42"}

ProtT3/all_checkpoints/stage2_07041521/wandb/run-20250704_152253-wgyb9m42/logs/debug.log

@@ -0,0 +1,24 @@
+2025-07-04 15:22:53,343 INFO    MainThread:41518 [wandb_setup.py:_flush():70] Current SDK version is 0.19.11
+2025-07-04 15:22:53,343 INFO    MainThread:41518 [wandb_setup.py:_flush():70] Configure stats pid to 41518
+2025-07-04 15:22:53,343 INFO    MainThread:41518 [wandb_setup.py:_flush():70] Loading settings from /root/.config/wandb/settings
+2025-07-04 15:22:53,343 INFO    MainThread:41518 [wandb_setup.py:_flush():70] Loading settings from /nas/shared/kilab/wangyujia/ProtT3/wandb/settings
+2025-07-04 15:22:53,343 INFO    MainThread:41518 [wandb_setup.py:_flush():70] Loading settings from environment variables
+2025-07-04 15:22:53,343 INFO    MainThread:41518 [wandb_init.py:setup_run_log_directory():724] Logging user logs to ./all_checkpoints/stage2_07041521/wandb/run-20250704_152253-wgyb9m42/logs/debug.log
+2025-07-04 15:22:53,343 INFO    MainThread:41518 [wandb_init.py:setup_run_log_directory():725] Logging internal logs to ./all_checkpoints/stage2_07041521/wandb/run-20250704_152253-wgyb9m42/logs/debug-internal.log
+2025-07-04 15:22:53,344 INFO    MainThread:41518 [wandb_init.py:init():852] calling init triggers
+2025-07-04 15:22:53,344 INFO    MainThread:41518 [wandb_init.py:init():857] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-07-04 15:22:53,344 INFO    MainThread:41518 [wandb_init.py:init():893] starting backend
+2025-07-04 15:22:53,344 INFO    MainThread:41518 [wandb_init.py:init():897] sending inform_init request
+2025-07-04 15:22:53,346 INFO    MainThread:41518 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-07-04 15:22:53,346 INFO    MainThread:41518 [wandb_init.py:init():907] backend started and connected
+2025-07-04 15:22:53,353 INFO    MainThread:41518 [wandb_init.py:init():1005] updated telemetry
+2025-07-04 15:22:53,357 INFO    MainThread:41518 [wandb_init.py:init():1029] communicating run to backend with 90.0 second timeout
+2025-07-04 15:22:58,087 INFO    MainThread:41518 [wandb_init.py:init():1104] starting run threads in backend
+2025-07-04 15:22:58,276 INFO    MainThread:41518 [wandb_run.py:_console_start():2573] atexit reg
+2025-07-04 15:22:58,276 INFO    MainThread:41518 [wandb_run.py:_redirect():2421] redirect: wrap_raw
+2025-07-04 15:22:58,282 INFO    MainThread:41518 [wandb_run.py:_redirect():2490] Wrapping output streams.
+2025-07-04 15:22:58,282 INFO    MainThread:41518 [wandb_run.py:_redirect():2513] Redirects installed.
+2025-07-04 15:22:58,284 INFO    MainThread:41518 [wandb_init.py:init():1150] run started, returning control to user process
+2025-07-04 15:23:06,942 INFO    MainThread:41518 [wandb_run.py:_config_callback():1436] config_cb None None {'filename': 'stage2_07041521', 'seed': 42, 'mode': 'train', 'strategy': 'deepspeed', 'accelerator': 'gpu', 'devices': '0,1,2,3,4,5,6,7', 'precision': 'bf16-mixed', 'max_epochs': 15, 'accumulate_grad_batches': 1, 'check_val_every_n_epoch': 1, 'enable_flash': False, 'use_wandb_logger': True, 'mix_dataset': True, 'save_every_n_epochs': 5, 'bert_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft', 'cross_attention_freq': 2, 'num_query_token': 8, 'llm_name': '/oss/wangyujia/BIO/pretrain_output/qwen2.5-7b-instruct-bio/bio_all/save1epoch/checkpoint-1300', 'num_beams': 5, 'do_sample': False, 'max_inference_len': 128, 'min_inference_len': 1, 'llm_tune': 'mid_lora', 'peft_config': '', 'peft_dir': '', 'plm_model': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m', 'plm_tune': 'freeze', 'lora_r': 8, 'lora_alpha': 16, 'lora_dropout': 0.1, 'enbale_gradient_checkpointing': False, 'weight_decay': 0.05, 'init_lr': 0.0001, 'min_lr': 1e-05, 'warmup_lr': 1e-06, 'warmup_steps': 1000, 'lr_decay_rate': 0.9, 'scheduler': 'linear_warmup_cosine_lr', 'stage1_path': '/nas/shared/kilab/wangyujia/ProtT3/all_checkpoints/stage1_06290009_deepspeed/epoch=19.ckpt/converted.ckpt', 'stage2_path': '', 'init_checkpoint': '', 'caption_eval_epoch': 10, 'num_workers': 8, 'batch_size': 32, 'inference_batch_size': 4, 'root': 'data', 'text_max_len': 128, 'q_max_len': 29, 'a_max_len': 36, 'prot_max_len': 1024, 'prompt': 'The protein has the following properties: ', 'filter_side_qa': False}
+2025-07-04 15:23:14,819 INFO    MsgRouterThr:41518 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles.

ProtT3/all_checkpoints/stage2_07041521/wandb/run-20250704_153023-ggclx68a/files/config.yaml

@@ -0,0 +1,150 @@
+_wandb:
+    value:
+        cli_version: 0.19.11
+        m:
+            - "1": trainer/global_step
+              "6":
+                - 3
+              "7": []
+        python_version: 3.10.0
+        t:
+            "1":
+                - 1
+                - 5
+                - 9
+                - 11
+                - 33
+                - 41
+                - 49
+                - 53
+                - 55
+                - 63
+                - 103
+            "2":
+                - 1
+                - 5
+                - 9
+                - 11
+                - 33
+                - 41
+                - 49
+                - 53
+                - 55
+                - 63
+                - 103
+            "3":
+                - 7
+                - 23
+                - 55
+                - 66
+            "4": 3.10.0
+            "5": 0.19.11
+            "6": 4.52.3
+            "8":
+                - 5
+            "12": 0.19.11
+            "13": linux-x86_64
+a_max_len:
+    value: 36
+accelerator:
+    value: gpu
+accumulate_grad_batches:
+    value: 1
+batch_size:
+    value: 32
+bert_name:
+    value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft
+caption_eval_epoch:
+    value: 10
+check_val_every_n_epoch:
+    value: 1
+cross_attention_freq:
+    value: 2
+devices:
+    value: 0,1,2,3,4,5,6,7
+do_sample:
+    value: false
+enable_flash:
+    value: false
+enbale_gradient_checkpointing:
+    value: false
+filename:
+    value: stage2_07041521
+filter_side_qa:
+    value: false
+inference_batch_size:
+    value: 4
+init_checkpoint:
+    value: ""
+init_lr:
+    value: 0.0001
+llm_name:
+    value: /oss/wangyujia/BIO/pretrain_output/qwen2.5-7b-instruct-bio/bio_all/save1epoch/checkpoint-1300
+llm_tune:
+    value: mid_lora
+lora_alpha:
+    value: 16
+lora_dropout:
+    value: 0.1
+lora_r:
+    value: 8
+lr_decay_rate:
+    value: 0.9
+max_epochs:
+    value: 15
+max_inference_len:
+    value: 128
+min_inference_len:
+    value: 1
+min_lr:
+    value: 1e-05
+mix_dataset:
+    value: true
+mode:
+    value: train
+num_beams:
+    value: 5
+num_query_token:
+    value: 8
+num_workers:
+    value: 8
+peft_config:
+    value: ""
+peft_dir:
+    value: ""
+plm_model:
+    value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m
+plm_tune:
+    value: freeze
+precision:
+    value: bf16-mixed
+prompt:
+    value: 'The protein has the following properties: '
+prot_max_len:
+    value: 1024
+q_max_len:
+    value: 29
+root:
+    value: data
+save_every_n_epochs:
+    value: 5
+scheduler:
+    value: linear_warmup_cosine_lr
+seed:
+    value: 42
+stage1_path:
+    value: /nas/shared/kilab/wangyujia/ProtT3/all_checkpoints/stage1_06290009_deepspeed/epoch=19.ckpt/converted.ckpt
+stage2_path:
+    value: ""
+strategy:
+    value: deepspeed
+text_max_len:
+    value: 128
+use_wandb_logger:
+    value: true
+warmup_lr:
+    value: 1e-06
+warmup_steps:
+    value: 1000
+weight_decay:
+    value: 0.05

ProtT3/all_checkpoints/stage2_07041521/wandb/run-20250704_153023-ggclx68a/files/output.log

@@ -0,0 +1,116 @@
+/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:654: Checkpoint directory /nas/shared/kilab/wangyujia/ProtT3/all_checkpoints/stage2_07041521 exists and is not empty.
+Enabling DeepSpeed BF16. Model parameters and inputs will be cast to `bfloat16`.
+LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]
+
+  | Name  | Type     | Params | Mode
+-------------------------------------------
+0 | blip2 | Blip2OPT | 7.9 B  | train
+-------------------------------------------
+104 M     Trainable params
+7.8 B     Non-trainable params
+7.9 B     Total params
+31,459.025Total estimated model params size (MB)
+174       Modules in train mode
+1203      Modules in eval mode
+Sanity Checking: |                                                                | 0/? [00:00<?, ?it/s]
+Traceback (most recent call last):
+  File "/nas/shared/kilab/wangyujia/ProtT3/stage2.py", line 130, in <module>
+    main(get_args())
+  File "/nas/shared/kilab/wangyujia/ProtT3/stage2.py", line 92, in main
+    trainer.fit(model, datamodule=dm)#, ckpt_path=args.ckpt_path)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 561, in fit
+    call._call_and_handle_interrupt(
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 47, in _call_and_handle_interrupt
+    return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 105, in launch
+    return function(*args, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 599, in _fit_impl
+    self._run(model, ckpt_path=ckpt_path)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1012, in _run
+    results = self._run_stage()
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1054, in _run_stage
+    self._run_sanity_check()
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1083, in _run_sanity_check
+    val_loop.run()
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/loops/utilities.py", line 179, in _decorator
+    return loop_run(self, *args, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/loops/evaluation_loop.py", line 138, in run
+    batch, batch_idx, dataloader_idx = next(data_fetcher)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/loops/fetchers.py", line 134, in __next__
+    batch = super().__next__()
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/loops/fetchers.py", line 61, in __next__
+    batch = next(self.iterator)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/utilities/combined_loader.py", line 341, in __next__
+    out = next(self._iterator)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/utilities/combined_loader.py", line 142, in __next__
+    out = next(self.iterators[0])
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 708, in __next__
+    data = self._next_data()
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1480, in _next_data
+    return self._process_data(data)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1505, in _process_data
+    data.reraise()
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/_utils.py", line 733, in reraise
+    raise exception
+NameError: Caught NameError in DataLoader worker process 0.
+Original Traceback (most recent call last):
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 349, in _worker_loop
+    data = fetcher.fetch(index)  # type: ignore[possibly-undefined]
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 52, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 52, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/nas/shared/kilab/wangyujia/ProtT3/data_provider/stage1_dm.py", line 397, in __getitem__
+    print(return_prompt)
+NameError: name 'return_prompt' is not defined
+
+[rank0]: Traceback (most recent call last):
+[rank0]:   File "/nas/shared/kilab/wangyujia/ProtT3/stage2.py", line 130, in <module>
+[rank0]:     main(get_args())
+[rank0]:   File "/nas/shared/kilab/wangyujia/ProtT3/stage2.py", line 92, in main
+[rank0]:     trainer.fit(model, datamodule=dm)#, ckpt_path=args.ckpt_path)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 561, in fit
+[rank0]:     call._call_and_handle_interrupt(
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 47, in _call_and_handle_interrupt
+[rank0]:     return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 105, in launch
+[rank0]:     return function(*args, **kwargs)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 599, in _fit_impl
+[rank0]:     self._run(model, ckpt_path=ckpt_path)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1012, in _run
+[rank0]:     results = self._run_stage()
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1054, in _run_stage
+[rank0]:     self._run_sanity_check()
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1083, in _run_sanity_check
+[rank0]:     val_loop.run()
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/loops/utilities.py", line 179, in _decorator
+[rank0]:     return loop_run(self, *args, **kwargs)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/loops/evaluation_loop.py", line 138, in run
+[rank0]:     batch, batch_idx, dataloader_idx = next(data_fetcher)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/loops/fetchers.py", line 134, in __next__
+[rank0]:     batch = super().__next__()
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/loops/fetchers.py", line 61, in __next__
+[rank0]:     batch = next(self.iterator)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/utilities/combined_loader.py", line 341, in __next__
+[rank0]:     out = next(self._iterator)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/utilities/combined_loader.py", line 142, in __next__
+[rank0]:     out = next(self.iterators[0])
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 708, in __next__
+[rank0]:     data = self._next_data()
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1480, in _next_data
+[rank0]:     return self._process_data(data)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1505, in _process_data
+[rank0]:     data.reraise()
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/_utils.py", line 733, in reraise
+[rank0]:     raise exception
+[rank0]: NameError: Caught NameError in DataLoader worker process 0.
+[rank0]: Original Traceback (most recent call last):
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 349, in _worker_loop
+[rank0]:     data = fetcher.fetch(index)  # type: ignore[possibly-undefined]
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 52, in fetch
+[rank0]:     data = [self.dataset[idx] for idx in possibly_batched_index]
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 52, in <listcomp>
+[rank0]:     data = [self.dataset[idx] for idx in possibly_batched_index]
+[rank0]:   File "/nas/shared/kilab/wangyujia/ProtT3/data_provider/stage1_dm.py", line 397, in __getitem__
+[rank0]:     print(return_prompt)
+[rank0]: NameError: name 'return_prompt' is not defined

ProtT3/all_checkpoints/stage2_07041521/wandb/run-20250704_153023-ggclx68a/files/requirements.txt

@@ -0,0 +1,225 @@
+pydantic_core==2.33.2
+psutil==7.0.0
+nvidia-cuda-nvrtc-cu12==12.4.127
+mpmath==1.3.0
+tzdata==2025.2
+contexttimer==0.3.3
+cycler==0.12.1
+python-magic==0.4.27
+pexpect==4.9.0
+sympy==1.13.1
+wrapt==1.17.2
+marisa-trie==1.2.1
+langcodes==3.5.0
+nvidia-nvtx-cu12==12.4.127
+ipython==8.36.0
+opencv-python-headless==4.5.5.64
+MarkupSafe==3.0.2
+jsonschema-specifications==2025.4.1
+wasabi==1.1.3
+blinker==1.9.0
+cfgv==3.4.0
+numpy==2.2.6
+idna==3.10
+nvidia-cufile-cu12==1.11.1.6
+ninja==1.11.1.4
+nvidia-nccl-cu12==2.21.5
+networkx==3.4.2
+certifi==2025.4.26
+deepspeed==0.16.10+b666844f
+pure_eval==0.2.3
+packaging==24.2
+nltk==3.9.1
+contourpy==1.3.2
+pre_commit==4.2.0
+nodeenv==1.9.1
+setuptools==78.1.1
+annotated-types==0.7.0
+multidict==6.4.4
+requests==2.32.3
+tornado==6.5.1
+triton==3.2.0
+pillow==11.2.1
+decord==0.6.0
+shellingham==1.5.4
+streamlit==1.45.1
+pydeck==0.9.1
+confection==0.1.5
+exceptiongroup==1.3.0
+prompt_toolkit==3.0.51
+text-unidecode==1.3
+nvidia-cufft-cu12==11.2.1.3
+antlr4-python3-runtime==4.9.3
+fairscale==0.4.4
+rouge_score==0.1.2
+nvidia-cudnn-cu12==9.1.0.70
+tqdm==4.67.1
+rich==14.0.0
+frozenlist==1.6.0
+webencodings==0.5.1
+altair==5.5.0
+opendatasets==0.1.22
+nvidia-curand-cu12==10.3.5.147
+protobuf==6.31.0
+asttokens==3.0.0
+wheel==0.45.1
+hf-xet==1.1.2
+weasel==0.4.1
+aiosignal==1.3.2
+absl-py==2.2.2
+thinc==8.3.6
+torchvision==0.21.0
+pandas==2.2.3
+fonttools==4.58.0
+bleach==6.2.0
+typing-inspection==0.4.1
+ftfy==6.3.1
+typing_extensions==4.13.2
+nvidia-ml-py==12.575.51
+python-slugify==8.0.4
+lightning-utilities==0.14.3
+py-cpuinfo==9.0.0
+smmap==5.0.2
+regex==2024.11.6
+scikit-image==0.25.2
+iopath==0.1.10
+spacy-legacy==3.0.12
+hjson==3.1.0
+executing==2.2.0
+kiwisolver==1.4.8
+scipy==1.15.3
+aiohappyeyeballs==2.6.1
+toml==0.10.2
+jedi==0.19.2
+GitPython==3.1.44
+ptyprocess==0.7.0
+kaggle==1.7.4.5
+braceexpand==0.1.7
+wcwidth==0.2.13
+nvidia-cuda-runtime-cu12==12.4.127
+pytorch-lightning==2.5.1.post0
+Jinja2==3.1.6
+urllib3==2.4.0
+watchdog==6.0.0
+filelock==3.18.0
+propcache==0.3.1
+torch==2.6.0
+nvidia-cusparse-cu12==12.3.1.170
+cymem==2.0.11
+nvidia-cusolver-cu12==11.6.1.9
+murmurhash==1.0.13
+catalogue==2.0.10
+yarl==1.20.0
+charset-normalizer==3.4.2
+gitdb==4.0.12
+matplotlib==3.10.3
+portalocker==3.1.1
+platformdirs==4.3.8
+async-timeout==5.0.1
+parso==0.8.4
+markdown-it-py==3.0.0
+omegaconf==2.3.0
+cloudpathlib==0.21.1
+nvidia-cusparselt-cu12==0.6.2
+spacy-loggers==1.0.5
+srsly==2.5.1
+identify==2.6.12
+rpds-py==0.25.1
+spacy==3.8.7
+matplotlib-inline==0.1.7
+smart-open==7.1.0
+pydantic==2.11.5
+mdurl==0.1.2
+virtualenv==20.31.2
+pytz==2025.2
+pycocotools==2.0.8
+six==1.17.0
+decorator==5.2.1
+referencing==0.36.2
+sentencepiece==0.2.0
+PyYAML==6.0.2
+pycocoevalcap==1.2
+imageio==2.37.0
+distlib==0.3.9
+pyarrow==20.0.0
+tenacity==9.1.2
+language_data==1.3.0
+nvidia-cuda-cupti-cu12==12.4.127
+blis==1.3.0
+Pygments==2.19.1
+tifffile==2025.5.10
+pyparsing==3.2.3
+cachetools==5.5.2
+safetensors==0.5.3
+attrs==25.3.0
+webdataset==0.2.111
+plotly==6.1.1
+nvidia-cublas-cu12==12.4.5.8
+timm==0.4.12
+torchmetrics==1.7.1
+nvidia-nvjitlink-cu12==12.4.127
+stack-data==0.6.3
+python-dateutil==2.9.0.post0
+lazy_loader==0.4
+traitlets==5.14.3
+einops==0.8.1
+salesforce-lavis==1.0.2
+joblib==1.5.1
+msgpack==1.1.0
+tokenizers==0.21.1
+sentry-sdk==2.29.1
+oss2==2.15.0
+setproctitle==1.3.6
+pip==25.1.1
+cffi==1.17.1
+transformers==4.52.3
+narwhals==1.41.0
+aliyun-python-sdk-core==2.16.0
+jsonschema==4.24.0
+flash-attn==2.7.1.post1
+preshed==3.0.10
+multiprocess==0.70.16
+cryptography==45.0.3
+aliyun-python-sdk-kms==2.16.5
+scikit-learn==1.6.1
+huggingface-hub==0.32.1
+crcmod==1.7
+typer==0.16.0
+web.py==0.62
+docker-pycreds==0.4.0
+xxhash==3.5.0
+bigmodelvis==0.0.1
+datasets==3.6.0
+more-itertools==10.7.0
+yacs==0.1.8
+jmespath==0.10.0
+aiohttp==3.12.2
+opencv-python==4.11.0.86
+pycparser==2.22
+threadpoolctl==3.6.0
+jaraco.functools==4.1.0
+click==8.2.1
+wandb==0.19.11
+opendelta==0.3.2
+pycryptodome==3.23.0
+pathlib==1.0.1
+dill==0.3.8
+fsspec==2025.3.0
+delta-center-client==0.0.4
+cheroot==10.0.1
+typing_extensions==4.12.2
+platformdirs==4.2.2
+jaraco.text==3.12.1
+packaging==24.2
+inflect==7.3.1
+jaraco.context==5.3.0
+wheel==0.45.1
+typeguard==4.3.0
+more-itertools==10.3.0
+tomli==2.0.1
+importlib_metadata==8.0.0
+backports.tarfile==1.2.0
+zipp==3.19.2
+jaraco.collections==5.1.0
+autocommand==2.2.2
+jaraco.functools==4.0.1

ProtT3/all_checkpoints/stage2_07041521/wandb/run-20250704_153023-ggclx68a/files/wandb-metadata.json

@@ -0,0 +1,104 @@
+{
+  "os": "Linux-5.10.134-008.16.kangaroo.al8.x86_64-x86_64-with-glibc2.35",
+  "python": "CPython 3.10.0",
+  "startedAt": "2025-07-04T07:30:24.142799Z",
+  "args": [
+    "--devices",
+    "0,1,2,3,4,5,6,7",
+    "--mode",
+    "train",
+    "--filename",
+    "stage2_07041521",
+    "--num_query_token",
+    "8",
+    "--save_every_n_epochs",
+    "5",
+    "--max_epochs",
+    "15",
+    "--batch_size",
+    "32",
+    "--precision",
+    "bf16-mixed",
+    "--num_workers",
+    "8",
+    "--plm_model",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m",
+    "--bert_name",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft",
+    "--llm_name",
+    "/oss/wangyujia/BIO/pretrain_output/qwen2.5-7b-instruct-bio/bio_all/save1epoch/checkpoint-1300",
+    "--llm_tune",
+    "mid_lora",
+    "--mix_dataset",
+    "--stage1_path",
+    "/nas/shared/kilab/wangyujia/ProtT3/all_checkpoints/stage1_06290009_deepspeed/epoch=19.ckpt/converted.ckpt",
+    "--use_wandb_logger"
+  ],
+  "program": "/nas/shared/kilab/wangyujia/ProtT3/stage2.py",
+  "codePath": "stage2.py",
+  "email": "gia0603yucca@gmail.com",
+  "root": "./all_checkpoints/stage2_07041521/",
+  "host": "dsw-265304-b8d7644bb-bs7r7",
+  "executable": "/root/miniconda3/envs/protT3/bin/python",
+  "codePathLocal": "stage2.py",
+  "cpu_count": 64,
+  "cpu_count_logical": 64,
+  "gpu": "NVIDIA A800-SXM4-80GB",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1623302262784",
+      "used": "1266515968"
+    }
+  },
+  "memory": {
+    "total": "549755813888"
+  },
+  "cpu": {
+    "count": 64,
+    "countLogical": 64
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    }
+  ],
+  "cudaVersion": "12.1"
+}

ProtT3/all_checkpoints/stage2_07041521/wandb/run-20250704_153023-ggclx68a/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":23}}

ProtT3/all_checkpoints/stage2_07041521/wandb/run-20250704_153023-ggclx68a/logs/debug-internal.log

@@ -0,0 +1,15 @@
+{"time":"2025-07-04T15:30:24.192995842+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.11","symlink path":"all_checkpoints/stage2_07041521/wandb/run-20250704_153023-ggclx68a/logs/debug-core.log"}
+{"time":"2025-07-04T15:30:27.170454248+08:00","level":"INFO","msg":"created new stream","id":"ggclx68a"}
+{"time":"2025-07-04T15:30:27.170510911+08:00","level":"INFO","msg":"stream: started","id":"ggclx68a"}
+{"time":"2025-07-04T15:30:27.170588716+08:00","level":"INFO","msg":"handler: started","stream_id":"ggclx68a"}
+{"time":"2025-07-04T15:30:27.170589658+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"ggclx68a"}
+{"time":"2025-07-04T15:30:27.170662113+08:00","level":"INFO","msg":"sender: started","stream_id":"ggclx68a"}
+{"time":"2025-07-04T15:30:30.523167583+08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2025-07-04T15:30:47.370064277+08:00","level":"INFO","msg":"stream: closing","id":"ggclx68a"}
+{"time":"2025-07-04T15:30:47.370150308+08:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2025-07-04T15:30:47.378162244+08:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2025-07-04T15:30:50.622126954+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-07-04T15:30:52.339883158+08:00","level":"INFO","msg":"handler: closed","stream_id":"ggclx68a"}
+{"time":"2025-07-04T15:30:52.339922114+08:00","level":"INFO","msg":"sender: closed","stream_id":"ggclx68a"}
+{"time":"2025-07-04T15:30:52.33991556+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"ggclx68a"}
+{"time":"2025-07-04T15:30:52.350227845+08:00","level":"INFO","msg":"stream: closed","id":"ggclx68a"}

ProtT3/all_checkpoints/stage2_07041521/wandb/run-20250704_153023-ggclx68a/logs/debug.log

@@ -0,0 +1,24 @@
+2025-07-04 15:30:24,140 INFO    MainThread:46730 [wandb_setup.py:_flush():70] Current SDK version is 0.19.11
+2025-07-04 15:30:24,140 INFO    MainThread:46730 [wandb_setup.py:_flush():70] Configure stats pid to 46730
+2025-07-04 15:30:24,140 INFO    MainThread:46730 [wandb_setup.py:_flush():70] Loading settings from /root/.config/wandb/settings
+2025-07-04 15:30:24,140 INFO    MainThread:46730 [wandb_setup.py:_flush():70] Loading settings from /nas/shared/kilab/wangyujia/ProtT3/wandb/settings
+2025-07-04 15:30:24,140 INFO    MainThread:46730 [wandb_setup.py:_flush():70] Loading settings from environment variables
+2025-07-04 15:30:24,140 INFO    MainThread:46730 [wandb_init.py:setup_run_log_directory():724] Logging user logs to ./all_checkpoints/stage2_07041521/wandb/run-20250704_153023-ggclx68a/logs/debug.log
+2025-07-04 15:30:24,140 INFO    MainThread:46730 [wandb_init.py:setup_run_log_directory():725] Logging internal logs to ./all_checkpoints/stage2_07041521/wandb/run-20250704_153023-ggclx68a/logs/debug-internal.log
+2025-07-04 15:30:24,140 INFO    MainThread:46730 [wandb_init.py:init():852] calling init triggers
+2025-07-04 15:30:24,140 INFO    MainThread:46730 [wandb_init.py:init():857] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-07-04 15:30:24,140 INFO    MainThread:46730 [wandb_init.py:init():893] starting backend
+2025-07-04 15:30:24,140 INFO    MainThread:46730 [wandb_init.py:init():897] sending inform_init request
+2025-07-04 15:30:24,142 INFO    MainThread:46730 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-07-04 15:30:24,142 INFO    MainThread:46730 [wandb_init.py:init():907] backend started and connected
+2025-07-04 15:30:24,143 INFO    MainThread:46730 [wandb_init.py:init():1005] updated telemetry
+2025-07-04 15:30:24,144 INFO    MainThread:46730 [wandb_init.py:init():1029] communicating run to backend with 90.0 second timeout
+2025-07-04 15:30:30,435 INFO    MainThread:46730 [wandb_init.py:init():1104] starting run threads in backend
+2025-07-04 15:30:30,681 INFO    MainThread:46730 [wandb_run.py:_console_start():2573] atexit reg
+2025-07-04 15:30:30,681 INFO    MainThread:46730 [wandb_run.py:_redirect():2421] redirect: wrap_raw
+2025-07-04 15:30:30,687 INFO    MainThread:46730 [wandb_run.py:_redirect():2490] Wrapping output streams.
+2025-07-04 15:30:30,692 INFO    MainThread:46730 [wandb_run.py:_redirect():2513] Redirects installed.
+2025-07-04 15:30:30,698 INFO    MainThread:46730 [wandb_init.py:init():1150] run started, returning control to user process
+2025-07-04 15:30:39,328 INFO    MainThread:46730 [wandb_run.py:_config_callback():1436] config_cb None None {'filename': 'stage2_07041521', 'seed': 42, 'mode': 'train', 'strategy': 'deepspeed', 'accelerator': 'gpu', 'devices': '0,1,2,3,4,5,6,7', 'precision': 'bf16-mixed', 'max_epochs': 15, 'accumulate_grad_batches': 1, 'check_val_every_n_epoch': 1, 'enable_flash': False, 'use_wandb_logger': True, 'mix_dataset': True, 'save_every_n_epochs': 5, 'bert_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft', 'cross_attention_freq': 2, 'num_query_token': 8, 'llm_name': '/oss/wangyujia/BIO/pretrain_output/qwen2.5-7b-instruct-bio/bio_all/save1epoch/checkpoint-1300', 'num_beams': 5, 'do_sample': False, 'max_inference_len': 128, 'min_inference_len': 1, 'llm_tune': 'mid_lora', 'peft_config': '', 'peft_dir': '', 'plm_model': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m', 'plm_tune': 'freeze', 'lora_r': 8, 'lora_alpha': 16, 'lora_dropout': 0.1, 'enbale_gradient_checkpointing': False, 'weight_decay': 0.05, 'init_lr': 0.0001, 'min_lr': 1e-05, 'warmup_lr': 1e-06, 'warmup_steps': 1000, 'lr_decay_rate': 0.9, 'scheduler': 'linear_warmup_cosine_lr', 'stage1_path': '/nas/shared/kilab/wangyujia/ProtT3/all_checkpoints/stage1_06290009_deepspeed/epoch=19.ckpt/converted.ckpt', 'stage2_path': '', 'init_checkpoint': '', 'caption_eval_epoch': 10, 'num_workers': 8, 'batch_size': 32, 'inference_batch_size': 4, 'root': 'data', 'text_max_len': 128, 'q_max_len': 29, 'a_max_len': 36, 'prot_max_len': 1024, 'prompt': 'The protein has the following properties: ', 'filter_side_qa': False}
+2025-07-04 15:30:47,369 INFO    MsgRouterThr:46730 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles.

ProtT3/all_checkpoints/stage2_07041521/wandb/run-20250704_153250-690krh73/files/config.yaml

@@ -0,0 +1,150 @@
+_wandb:
+    value:
+        cli_version: 0.19.11
+        m:
+            - "1": trainer/global_step
+              "6":
+                - 3
+              "7": []
+        python_version: 3.10.0
+        t:
+            "1":
+                - 1
+                - 5
+                - 9
+                - 11
+                - 33
+                - 41
+                - 49
+                - 53
+                - 55
+                - 63
+                - 103
+            "2":
+                - 1
+                - 5
+                - 9
+                - 11
+                - 33
+                - 41
+                - 49
+                - 53
+                - 55
+                - 63
+                - 103
+            "3":
+                - 7
+                - 23
+                - 55
+                - 66
+            "4": 3.10.0
+            "5": 0.19.11
+            "6": 4.52.3
+            "8":
+                - 5
+            "12": 0.19.11
+            "13": linux-x86_64
+a_max_len:
+    value: 36
+accelerator:
+    value: gpu
+accumulate_grad_batches:
+    value: 1
+batch_size:
+    value: 32
+bert_name:
+    value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft
+caption_eval_epoch:
+    value: 10
+check_val_every_n_epoch:
+    value: 1
+cross_attention_freq:
+    value: 2
+devices:
+    value: 0,1,2,3,4,5,6,7
+do_sample:
+    value: false
+enable_flash:
+    value: false
+enbale_gradient_checkpointing:
+    value: false
+filename:
+    value: stage2_07041521
+filter_side_qa:
+    value: false
+inference_batch_size:
+    value: 4
+init_checkpoint:
+    value: ""
+init_lr:
+    value: 0.0001
+llm_name:
+    value: /oss/wangyujia/BIO/pretrain_output/qwen2.5-7b-instruct-bio/bio_all/save1epoch/checkpoint-1300
+llm_tune:
+    value: mid_lora
+lora_alpha:
+    value: 16
+lora_dropout:
+    value: 0.1
+lora_r:
+    value: 8
+lr_decay_rate:
+    value: 0.9
+max_epochs:
+    value: 15
+max_inference_len:
+    value: 128
+min_inference_len:
+    value: 1
+min_lr:
+    value: 1e-05
+mix_dataset:
+    value: true
+mode:
+    value: train
+num_beams:
+    value: 5
+num_query_token:
+    value: 8
+num_workers:
+    value: 8
+peft_config:
+    value: ""
+peft_dir:
+    value: ""
+plm_model:
+    value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m
+plm_tune:
+    value: freeze
+precision:
+    value: bf16-mixed
+prompt:
+    value: 'The protein has the following properties: '
+prot_max_len:
+    value: 1024
+q_max_len:
+    value: 29
+root:
+    value: data
+save_every_n_epochs:
+    value: 5
+scheduler:
+    value: linear_warmup_cosine_lr
+seed:
+    value: 42
+stage1_path:
+    value: /nas/shared/kilab/wangyujia/ProtT3/all_checkpoints/stage1_06290009_deepspeed/epoch=19.ckpt/converted.ckpt
+stage2_path:
+    value: ""
+strategy:
+    value: deepspeed
+text_max_len:
+    value: 128
+use_wandb_logger:
+    value: true
+warmup_lr:
+    value: 1e-06
+warmup_steps:
+    value: 1000
+weight_decay:
+    value: 0.05

ProtT3/all_checkpoints/stage2_07041521/wandb/run-20250704_153250-690krh73/files/output.log

@@ -0,0 +1,191 @@
+/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:654: Checkpoint directory /nas/shared/kilab/wangyujia/ProtT3/all_checkpoints/stage2_07041521 exists and is not empty.
+Enabling DeepSpeed BF16. Model parameters and inputs will be cast to `bfloat16`.
+LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]
+
+  | Name  | Type     | Params | Mode
+-------------------------------------------
+0 | blip2 | Blip2OPT | 7.9 B  | train
+-------------------------------------------
+104 M     Trainable params
+7.8 B     Non-trainable params
+7.9 B     Total params
+31,459.025Total estimated model params size (MB)
+174       Modules in train mode
+1203      Modules in eval mode
+Sanity Checking DataLoader 0:   0%|                                               | 0/2 [00:00<?, ?it/s]
+Traceback (most recent call last):
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 47, in _call_and_handle_interrupt
+    return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 105, in launch
+    return function(*args, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 599, in _fit_impl
+    self._run(model, ckpt_path=ckpt_path)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1012, in _run
+    results = self._run_stage()
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1054, in _run_stage
+    self._run_sanity_check()
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1083, in _run_sanity_check
+    val_loop.run()
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/loops/utilities.py", line 179, in _decorator
+    return loop_run(self, *args, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/loops/evaluation_loop.py", line 145, in run
+    self._evaluation_step(batch, batch_idx, dataloader_idx, dataloader_iter)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/loops/evaluation_loop.py", line 437, in _evaluation_step
+    output = call._call_strategy_hook(trainer, hook_name, *step_args)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 328, in _call_strategy_hook
+    output = fn(*args, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 411, in validation_step
+    return self._forward_redirection(self.model, self.lightning_module, "validation_step", *args, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 641, in __call__
+    wrapper_output = wrapper_module(*args, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/deepspeed/utils/nvtx.py", line 20, in wrapped_fn
+    ret_val = func(*args, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/deepspeed/runtime/engine.py", line 2054, in forward
+    loss = self.module(*inputs, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1845, in _call_impl
+    return inner()
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1793, in inner
+    result = forward_call(*args, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 634, in wrapped_forward
+    out = method(*_args, **_kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context
+    return func(*args, **kwargs)
+  File "/nas/shared/kilab/wangyujia/ProtT3/model/blip2_stage2.py", line 119, in validation_step
+    loss = self.blip2(batch)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/nas/shared/kilab/wangyujia/ProtT3/model/blip2_opt.py", line 222, in forward
+    prot_batch, prompt_batch, text_batch = batch
+ValueError: not enough values to unpack (expected 3, got 2)
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/nas/shared/kilab/wangyujia/ProtT3/stage2.py", line 130, in <module>
+    main(get_args())
+  File "/nas/shared/kilab/wangyujia/ProtT3/stage2.py", line 92, in main
+    trainer.fit(model, datamodule=dm)#, ckpt_path=args.ckpt_path)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 561, in fit
+    call._call_and_handle_interrupt(
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 69, in _call_and_handle_interrupt
+    trainer._teardown()
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1035, in _teardown
+    self.strategy.teardown()
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/strategies/ddp.py", line 419, in teardown
+    super().teardown()
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/strategies/parallel.py", line 134, in teardown
+    super().teardown()
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 536, in teardown
+    self.lightning_module.cpu()
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/lightning_fabric/utilities/device_dtype_mixin.py", line 82, in cpu
+    return super().cpu()
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1121, in cpu
+    return self._apply(lambda t: t.cpu())
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 903, in _apply
+    module._apply(fn)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 903, in _apply
+    module._apply(fn)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 903, in _apply
+    module._apply(fn)
+  [Previous line repeated 4 more times]
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 930, in _apply
+    param_applied = fn(param)
+  File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1121, in <lambda>
+    return self._apply(lambda t: t.cpu())
+KeyboardInterrupt
+[rank0]: Traceback (most recent call last):
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 47, in _call_and_handle_interrupt
+[rank0]:     return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 105, in launch
+[rank0]:     return function(*args, **kwargs)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 599, in _fit_impl
+[rank0]:     self._run(model, ckpt_path=ckpt_path)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1012, in _run
+[rank0]:     results = self._run_stage()
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1054, in _run_stage
+[rank0]:     self._run_sanity_check()
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1083, in _run_sanity_check
+[rank0]:     val_loop.run()
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/loops/utilities.py", line 179, in _decorator
+[rank0]:     return loop_run(self, *args, **kwargs)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/loops/evaluation_loop.py", line 145, in run
+[rank0]:     self._evaluation_step(batch, batch_idx, dataloader_idx, dataloader_iter)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/loops/evaluation_loop.py", line 437, in _evaluation_step
+[rank0]:     output = call._call_strategy_hook(trainer, hook_name, *step_args)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 328, in _call_strategy_hook
+[rank0]:     output = fn(*args, **kwargs)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 411, in validation_step
+[rank0]:     return self._forward_redirection(self.model, self.lightning_module, "validation_step", *args, **kwargs)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 641, in __call__
+[rank0]:     wrapper_output = wrapper_module(*args, **kwargs)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
+[rank0]:     return self._call_impl(*args, **kwargs)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
+[rank0]:     return forward_call(*args, **kwargs)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/deepspeed/utils/nvtx.py", line 20, in wrapped_fn
+[rank0]:     ret_val = func(*args, **kwargs)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/deepspeed/runtime/engine.py", line 2054, in forward
+[rank0]:     loss = self.module(*inputs, **kwargs)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
+[rank0]:     return self._call_impl(*args, **kwargs)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1845, in _call_impl
+[rank0]:     return inner()
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1793, in inner
+[rank0]:     result = forward_call(*args, **kwargs)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 634, in wrapped_forward
+[rank0]:     out = method(*_args, **_kwargs)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context
+[rank0]:     return func(*args, **kwargs)
+[rank0]:   File "/nas/shared/kilab/wangyujia/ProtT3/model/blip2_stage2.py", line 119, in validation_step
+[rank0]:     loss = self.blip2(batch)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
+[rank0]:     return self._call_impl(*args, **kwargs)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
+[rank0]:     return forward_call(*args, **kwargs)
+[rank0]:   File "/nas/shared/kilab/wangyujia/ProtT3/model/blip2_opt.py", line 222, in forward
+[rank0]:     prot_batch, prompt_batch, text_batch = batch
+[rank0]: ValueError: not enough values to unpack (expected 3, got 2)
+
+[rank0]: During handling of the above exception, another exception occurred:
+
+[rank0]: Traceback (most recent call last):
+[rank0]:   File "/nas/shared/kilab/wangyujia/ProtT3/stage2.py", line 130, in <module>
+[rank0]:     main(get_args())
+[rank0]:   File "/nas/shared/kilab/wangyujia/ProtT3/stage2.py", line 92, in main
+[rank0]:     trainer.fit(model, datamodule=dm)#, ckpt_path=args.ckpt_path)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 561, in fit
+[rank0]:     call._call_and_handle_interrupt(
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 69, in _call_and_handle_interrupt
+[rank0]:     trainer._teardown()
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1035, in _teardown
+[rank0]:     self.strategy.teardown()
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/strategies/ddp.py", line 419, in teardown
+[rank0]:     super().teardown()
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/strategies/parallel.py", line 134, in teardown
+[rank0]:     super().teardown()
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 536, in teardown
+[rank0]:     self.lightning_module.cpu()
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/lightning_fabric/utilities/device_dtype_mixin.py", line 82, in cpu
+[rank0]:     return super().cpu()
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1121, in cpu
+[rank0]:     return self._apply(lambda t: t.cpu())
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 903, in _apply
+[rank0]:     module._apply(fn)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 903, in _apply
+[rank0]:     module._apply(fn)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 903, in _apply
+[rank0]:     module._apply(fn)
+[rank0]:   [Previous line repeated 4 more times]
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 930, in _apply
+[rank0]:     param_applied = fn(param)
+[rank0]:   File "/root/miniconda3/envs/protT3/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1121, in <lambda>
+[rank0]:     return self._apply(lambda t: t.cpu())
+[rank0]: KeyboardInterrupt

ProtT3/all_checkpoints/stage2_07041521/wandb/run-20250704_153250-690krh73/files/requirements.txt

@@ -0,0 +1,225 @@
+pydantic_core==2.33.2
+psutil==7.0.0
+nvidia-cuda-nvrtc-cu12==12.4.127
+mpmath==1.3.0
+tzdata==2025.2
+contexttimer==0.3.3
+cycler==0.12.1
+python-magic==0.4.27
+pexpect==4.9.0
+sympy==1.13.1
+wrapt==1.17.2
+marisa-trie==1.2.1
+langcodes==3.5.0
+nvidia-nvtx-cu12==12.4.127
+ipython==8.36.0
+opencv-python-headless==4.5.5.64
+MarkupSafe==3.0.2
+jsonschema-specifications==2025.4.1
+wasabi==1.1.3
+blinker==1.9.0
+cfgv==3.4.0
+numpy==2.2.6
+idna==3.10
+nvidia-cufile-cu12==1.11.1.6
+ninja==1.11.1.4
+nvidia-nccl-cu12==2.21.5
+networkx==3.4.2
+certifi==2025.4.26
+deepspeed==0.16.10+b666844f
+pure_eval==0.2.3
+packaging==24.2
+nltk==3.9.1
+contourpy==1.3.2
+pre_commit==4.2.0
+nodeenv==1.9.1
+setuptools==78.1.1
+annotated-types==0.7.0
+multidict==6.4.4
+requests==2.32.3
+tornado==6.5.1
+triton==3.2.0
+pillow==11.2.1
+decord==0.6.0
+shellingham==1.5.4
+streamlit==1.45.1
+pydeck==0.9.1
+confection==0.1.5
+exceptiongroup==1.3.0
+prompt_toolkit==3.0.51
+text-unidecode==1.3
+nvidia-cufft-cu12==11.2.1.3
+antlr4-python3-runtime==4.9.3
+fairscale==0.4.4
+rouge_score==0.1.2
+nvidia-cudnn-cu12==9.1.0.70
+tqdm==4.67.1
+rich==14.0.0
+frozenlist==1.6.0
+webencodings==0.5.1
+altair==5.5.0
+opendatasets==0.1.22
+nvidia-curand-cu12==10.3.5.147
+protobuf==6.31.0
+asttokens==3.0.0
+wheel==0.45.1
+hf-xet==1.1.2
+weasel==0.4.1
+aiosignal==1.3.2
+absl-py==2.2.2
+thinc==8.3.6
+torchvision==0.21.0
+pandas==2.2.3
+fonttools==4.58.0
+bleach==6.2.0
+typing-inspection==0.4.1
+ftfy==6.3.1
+typing_extensions==4.13.2
+nvidia-ml-py==12.575.51
+python-slugify==8.0.4
+lightning-utilities==0.14.3
+py-cpuinfo==9.0.0
+smmap==5.0.2
+regex==2024.11.6
+scikit-image==0.25.2
+iopath==0.1.10
+spacy-legacy==3.0.12
+hjson==3.1.0
+executing==2.2.0
+kiwisolver==1.4.8
+scipy==1.15.3
+aiohappyeyeballs==2.6.1
+toml==0.10.2
+jedi==0.19.2
+GitPython==3.1.44
+ptyprocess==0.7.0
+kaggle==1.7.4.5
+braceexpand==0.1.7
+wcwidth==0.2.13
+nvidia-cuda-runtime-cu12==12.4.127
+pytorch-lightning==2.5.1.post0
+Jinja2==3.1.6
+urllib3==2.4.0
+watchdog==6.0.0
+filelock==3.18.0
+propcache==0.3.1
+torch==2.6.0
+nvidia-cusparse-cu12==12.3.1.170
+cymem==2.0.11
+nvidia-cusolver-cu12==11.6.1.9
+murmurhash==1.0.13
+catalogue==2.0.10
+yarl==1.20.0
+charset-normalizer==3.4.2
+gitdb==4.0.12
+matplotlib==3.10.3
+portalocker==3.1.1
+platformdirs==4.3.8
+async-timeout==5.0.1
+parso==0.8.4
+markdown-it-py==3.0.0
+omegaconf==2.3.0
+cloudpathlib==0.21.1
+nvidia-cusparselt-cu12==0.6.2
+spacy-loggers==1.0.5
+srsly==2.5.1
+identify==2.6.12
+rpds-py==0.25.1
+spacy==3.8.7
+matplotlib-inline==0.1.7
+smart-open==7.1.0
+pydantic==2.11.5
+mdurl==0.1.2
+virtualenv==20.31.2
+pytz==2025.2
+pycocotools==2.0.8
+six==1.17.0
+decorator==5.2.1
+referencing==0.36.2
+sentencepiece==0.2.0
+PyYAML==6.0.2
+pycocoevalcap==1.2
+imageio==2.37.0
+distlib==0.3.9
+pyarrow==20.0.0
+tenacity==9.1.2
+language_data==1.3.0
+nvidia-cuda-cupti-cu12==12.4.127
+blis==1.3.0
+Pygments==2.19.1
+tifffile==2025.5.10
+pyparsing==3.2.3
+cachetools==5.5.2
+safetensors==0.5.3
+attrs==25.3.0
+webdataset==0.2.111
+plotly==6.1.1
+nvidia-cublas-cu12==12.4.5.8
+timm==0.4.12
+torchmetrics==1.7.1
+nvidia-nvjitlink-cu12==12.4.127
+stack-data==0.6.3
+python-dateutil==2.9.0.post0
+lazy_loader==0.4
+traitlets==5.14.3
+einops==0.8.1
+salesforce-lavis==1.0.2
+joblib==1.5.1
+msgpack==1.1.0
+tokenizers==0.21.1
+sentry-sdk==2.29.1
+oss2==2.15.0
+setproctitle==1.3.6
+pip==25.1.1
+cffi==1.17.1
+transformers==4.52.3
+narwhals==1.41.0
+aliyun-python-sdk-core==2.16.0
+jsonschema==4.24.0
+flash-attn==2.7.1.post1
+preshed==3.0.10
+multiprocess==0.70.16
+cryptography==45.0.3
+aliyun-python-sdk-kms==2.16.5
+scikit-learn==1.6.1
+huggingface-hub==0.32.1
+crcmod==1.7
+typer==0.16.0
+web.py==0.62
+docker-pycreds==0.4.0
+xxhash==3.5.0
+bigmodelvis==0.0.1
+datasets==3.6.0
+more-itertools==10.7.0
+yacs==0.1.8
+jmespath==0.10.0
+aiohttp==3.12.2
+opencv-python==4.11.0.86
+pycparser==2.22
+threadpoolctl==3.6.0
+jaraco.functools==4.1.0
+click==8.2.1
+wandb==0.19.11
+opendelta==0.3.2
+pycryptodome==3.23.0
+pathlib==1.0.1
+dill==0.3.8
+fsspec==2025.3.0
+delta-center-client==0.0.4
+cheroot==10.0.1
+typing_extensions==4.12.2
+platformdirs==4.2.2
+jaraco.text==3.12.1
+packaging==24.2
+inflect==7.3.1
+jaraco.context==5.3.0
+wheel==0.45.1
+typeguard==4.3.0
+more-itertools==10.3.0
+tomli==2.0.1
+importlib_metadata==8.0.0
+backports.tarfile==1.2.0
+zipp==3.19.2
+jaraco.collections==5.1.0
+autocommand==2.2.2
+jaraco.functools==4.0.1

ProtT3/all_checkpoints/stage2_07041521/wandb/run-20250704_153250-690krh73/files/wandb-metadata.json

@@ -0,0 +1,104 @@
+{
+  "os": "Linux-5.10.134-008.16.kangaroo.al8.x86_64-x86_64-with-glibc2.35",
+  "python": "CPython 3.10.0",
+  "startedAt": "2025-07-04T07:32:50.809990Z",
+  "args": [
+    "--devices",
+    "0,1,2,3,4,5,6,7",
+    "--mode",
+    "train",
+    "--filename",
+    "stage2_07041521",
+    "--num_query_token",
+    "8",
+    "--save_every_n_epochs",
+    "5",
+    "--max_epochs",
+    "15",
+    "--batch_size",
+    "32",
+    "--precision",
+    "bf16-mixed",
+    "--num_workers",
+    "8",
+    "--plm_model",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m",
+    "--bert_name",
+    "/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft",
+    "--llm_name",
+    "/oss/wangyujia/BIO/pretrain_output/qwen2.5-7b-instruct-bio/bio_all/save1epoch/checkpoint-1300",
+    "--llm_tune",
+    "mid_lora",
+    "--mix_dataset",
+    "--stage1_path",
+    "/nas/shared/kilab/wangyujia/ProtT3/all_checkpoints/stage1_06290009_deepspeed/epoch=19.ckpt/converted.ckpt",
+    "--use_wandb_logger"
+  ],
+  "program": "/nas/shared/kilab/wangyujia/ProtT3/stage2.py",
+  "codePath": "stage2.py",
+  "email": "gia0603yucca@gmail.com",
+  "root": "./all_checkpoints/stage2_07041521/",
+  "host": "dsw-265304-b8d7644bb-bs7r7",
+  "executable": "/root/miniconda3/envs/protT3/bin/python",
+  "codePathLocal": "stage2.py",
+  "cpu_count": 64,
+  "cpu_count_logical": 64,
+  "gpu": "NVIDIA A800-SXM4-80GB",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1623302262784",
+      "used": "1266544640"
+    }
+  },
+  "memory": {
+    "total": "549755813888"
+  },
+  "cpu": {
+    "count": 64,
+    "countLogical": 64
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A800-SXM4-80GB",
+      "memoryTotal": "85198045184",
+      "architecture": "Ampere"
+    }
+  ],
+  "cudaVersion": "12.1"
+}

ProtT3/all_checkpoints/stage2_07041521/wandb/run-20250704_153250-690krh73/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":16}}

ProtT3/all_checkpoints/stage2_07041521/wandb/run-20250704_153250-690krh73/logs/debug-internal.log

@@ -0,0 +1,15 @@
+{"time":"2025-07-04T15:32:50.852026927+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.11","symlink path":"all_checkpoints/stage2_07041521/wandb/run-20250704_153250-690krh73/logs/debug-core.log"}
+{"time":"2025-07-04T15:32:52.202641273+08:00","level":"INFO","msg":"created new stream","id":"690krh73"}
+{"time":"2025-07-04T15:32:52.202708431+08:00","level":"INFO","msg":"handler: started","stream_id":"690krh73"}
+{"time":"2025-07-04T15:32:52.202727749+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"690krh73"}
+{"time":"2025-07-04T15:32:52.202728265+08:00","level":"INFO","msg":"stream: started","id":"690krh73"}
+{"time":"2025-07-04T15:32:52.202837136+08:00","level":"INFO","msg":"sender: started","stream_id":"690krh73"}
+{"time":"2025-07-04T15:32:53.598926263+08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2025-07-04T15:33:07.744102695+08:00","level":"INFO","msg":"stream: closing","id":"690krh73"}
+{"time":"2025-07-04T15:33:07.74416603+08:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2025-07-04T15:33:07.76280695+08:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2025-07-04T15:33:09.484600235+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-07-04T15:33:11.350265605+08:00","level":"INFO","msg":"handler: closed","stream_id":"690krh73"}
+{"time":"2025-07-04T15:33:11.35028236+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"690krh73"}
+{"time":"2025-07-04T15:33:11.350307246+08:00","level":"INFO","msg":"sender: closed","stream_id":"690krh73"}
+{"time":"2025-07-04T15:33:11.363954115+08:00","level":"INFO","msg":"stream: closed","id":"690krh73"}