Add files using upload-large-folder tool

BIO/ablation/test.py

@@ -0,0 +1,157 @@
+
+# #accuracy
+import json
+import re
+
+def compute_accuracy_from_file(filepath):
+    total, correct = 0, 0
+    pattern = re.compile(r"<ANSWER>(.*?)</ANSWER>", re.IGNORECASE)
+
+    with open(filepath, 'r', encoding='utf-8') as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            data = json.loads(line)
+            print(data)
+            tgt_match = data.get("reference_answer", "")
+            print(tgt_match)
+            pred_match = pattern.search(data.get("generated_answer", ""))
+            #pred_match = data.get("generated_answer", "")
+            print(pred_match)
+            if not tgt_match or not pred_match:
+                continue
+            
+            try:
+                tgt_val = int(tgt_match.strip())
+                pred_val = int(pred_match.group(1).strip())
+                 #.group(1).
+            except ValueError:
+                continue  # 如果无法转换为int，跳过此条
+
+
+            total += 1
+            if pred_val == tgt_val:
+                correct += 1
+
+    if total == 0:
+        return 0.0, 0
+    return correct / total, total
+
+if __name__ == "__main__":
+    filepath = "/nas/shared/kilab/wangyujia/BIO/ablation/temperature_stability.jsonl"
+    acc, count = compute_accuracy_from_file(filepath)
+    print(f"Checked {count} items. Accuracy: {acc*100:.3f}%")
+
+
+# #spearman
+# import json
+# import re
+# from scipy.stats import spearmanr
+
+# # 文件路径（替换为你的 JSONL 文件路径）
+# file_path = '/nas/shared/kilab/wangyujia/BIO/ablation/temperature_stability.jsonl'
+
+# # 正则：提取 <answer>...</answer> 中的数值
+# pattern = re.compile(r"<answer>(.*?)</answer>")
+
+# y_true = []  # reference_answer 中的真实值
+# y_pred = []  # generated_answer 预测值
+
+# with open(file_path, 'r', encoding='utf-8') as f:
+#     for line in f:
+#         line = line.strip()
+#         print(line)
+#         if not line:
+#             continue
+#         try:
+#             data = json.loads(line)  # 每行都是一个 JSON 对象
+#             reference_str = data.get("reference_answer", "").strip()
+#             generated_str = data.get("generated_answer", "").strip()
+#             print(reference_str)
+#             print(generated_str)
+#             # 提取 generated_answer 中的数值
+#             pred_match = pattern.search(generated_str)
+#             print(pred_match.group(1))
+#             if pred_match:
+#                 pred_value = float(pred_match.group(1))  # 提取 <answer> 中的值
+#                 true_value = float(reference_str)        # reference_answer 本身就是数值字符串
+#                 y_true.append(true_value)
+#                 y_pred.append(pred_value)
+#             else:
+#                 print(f"未找到 <answer> 标签，跳过：{generated_str}")
+#         except Exception as e:
+#             print(f"处理行时出错：{line}")
+#             print(f"错误：{e}")
+#             continue
+
+# # 计算 Spearman 相关系数
+# if len(y_true) > 1:
+#     print(len(y_true))
+#     print(len(y_pred))
+#     rho, p_value = spearmanr(y_pred,y_true)
+#     print(f"有效样本数：{len(y_true)}")
+#     print(f"Spearman correlation coefficient: {rho:.5f}, p-value: {p_value:.4e}")
+# else:
+#     print("有效数据不足，无法计算 Spearman 相关系数。")
+
+
+# # f1
+# import re
+# import json
+
+# def extract_numbers_from_generated_answer(s):
+#     """从<answer>...</answer>中提取纯数字"""
+#     match = re.search(r'<answer>(.*?)</answer>', s)
+#     if match:
+#         numbers_str = match.group(1)
+#         numbers = re.findall(r'\d+', numbers_str)
+#         return set(map(int, numbers))
+#     else:
+#         return set()
+
+# def extract_numbers_from_reference_answer(s):
+#     """从参考答案字符串中提取数字，忽略非数字字符"""
+#     numbers = re.findall(r'\d+', s)
+#     return set(map(int, numbers))
+
+# def calculate_f1(pred_set, target_set):
+#     if not pred_set and not target_set:
+#         return 1.0
+#     if not pred_set or not target_set:
+#         return 0.0
+#     tp = len(pred_set & target_set)
+#     fp = len(pred_set - target_set)
+#     fn = len(target_set - pred_set)
+#     precision = tp / (tp + fp) if (tp + fp) > 0 else 0.0
+#     recall = tp / (tp + fn) if (tp + fn) > 0 else 0.0
+#     if precision + recall == 0:
+#         return 0.0
+#     return 2 * precision * recall / (precision + recall)
+
+# # 文件路径
+# file_path = '/nas/shared/kilab/wangyujia/BIO/ablation/enzyme_commission_number.jsonl'
+
+# all_f1_scores = []
+
+# with open(file_path, 'r', encoding='utf-8') as f:
+#     for line in f:
+#         try:
+#             data = json.loads(line)
+#             reference = data.get('reference_answer', '')
+#             generated = data.get('generated_answer', '')
+#             target_set = extract_numbers_from_reference_answer(reference)
+#             pred_set = extract_numbers_from_generated_answer(generated)
+#             f1 = calculate_f1(pred_set, target_set)
+#             all_f1_scores.append(f1)
+#         except Exception as e:
+#             print(f"处理行时出错：{line.strip()}")
+#             print(f"错误：{e}")
+
+# # 输出结果
+# if all_f1_scores:
+#     avg_f1 = sum(all_f1_scores) / len(all_f1_scores)
+#     print(f"总样本数：{len(all_f1_scores)}")
+#     print(f"平均 F1 分数：{avg_f1:.5f}")
+# else:
+#     print("没有有效的F1分数可供计算")

BIO/sft/qwen-metal_ion_binding-08022141/v0-20250802-215822/args.json

@@ -0,0 +1,364 @@
+{
+  "model": "/oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged",
+  "model_type": "qwen2_5",
+  "model_revision": null,
+  "task_type": "causal_lm",
+  "torch_dtype": "bfloat16",
+  "attn_impl": null,
+  "num_labels": null,
+  "problem_type": null,
+  "rope_scaling": null,
+  "device_map": null,
+  "max_memory": {},
+  "local_repo_path": null,
+  "template": "qwen2_5",
+  "system": null,
+  "max_length": 8192,
+  "truncation_strategy": "delete",
+  "max_pixels": null,
+  "tools_prompt": "react_en",
+  "norm_bbox": null,
+  "response_prefix": null,
+  "padding_side": "right",
+  "loss_scale": "default",
+  "sequence_parallel_size": 1,
+  "use_chat_template": true,
+  "template_backend": "swift",
+  "dataset": [
+    "/oss/wangyujia/ProtT3/ProtT3/data/sft/dataset/metal_ion_binding/train.jsonl"
+  ],
+  "val_dataset": [],
+  "split_dataset_ratio": 0.01,
+  "data_seed": 42,
+  "dataset_num_proc": 128,
+  "dataset_shuffle": true,
+  "val_dataset_shuffle": false,
+  "streaming": false,
+  "interleave_prob": null,
+  "stopping_strategy": "first_exhausted",
+  "shuffle_buffer_size": 1000,
+  "enable_cache": false,
+  "download_mode": "reuse_dataset_if_exists",
+  "columns": {},
+  "strict": false,
+  "remove_unused_columns": true,
+  "model_name": [
+    "qwen_bio_sft_deeplocbinary-08022035"
+  ],
+  "model_author": [
+    "swift"
+  ],
+  "custom_dataset_info": [],
+  "quant_method": null,
+  "quant_bits": null,
+  "hqq_axis": null,
+  "bnb_4bit_compute_dtype": "bfloat16",
+  "bnb_4bit_quant_type": "nf4",
+  "bnb_4bit_use_double_quant": true,
+  "bnb_4bit_quant_storage": null,
+  "max_new_tokens": 64,
+  "temperature": 0.0,
+  "top_k": null,
+  "top_p": null,
+  "repetition_penalty": null,
+  "num_beams": 1,
+  "stream": false,
+  "stop_words": [],
+  "logprobs": false,
+  "top_logprobs": null,
+  "ckpt_dir": "/oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged",
+  "load_dataset_config": null,
+  "lora_modules": [],
+  "tuner_backend": "peft",
+  "train_type": "lora",
+  "adapters": [],
+  "external_plugins": [],
+  "seed": 42,
+  "model_kwargs": {},
+  "load_args": false,
+  "load_data_args": false,
+  "use_hf": false,
+  "hub_token": null,
+  "custom_register_path": [],
+  "ignore_args_error": false,
+  "use_swift_lora": false,
+  "output_dir": "/nas/shared/kilab/wangyujia/BIO/sft/qwen-metal_ion_binding-08022141/v0-20250802-215822",
+  "overwrite_output_dir": false,
+  "do_train": false,
+  "do_eval": false,
+  "do_predict": false,
+  "eval_strategy": "steps",
+  "prediction_loss_only": false,
+  "per_device_train_batch_size": 2,
+  "per_device_eval_batch_size": 2,
+  "per_gpu_train_batch_size": null,
+  "per_gpu_eval_batch_size": null,
+  "gradient_accumulation_steps": 4,
+  "eval_accumulation_steps": null,
+  "eval_delay": 0,
+  "torch_empty_cache_steps": null,
+  "learning_rate": 1e-05,
+  "weight_decay": 0.1,
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.95,
+  "adam_epsilon": 1e-08,
+  "max_grad_norm": 1.0,
+  "num_train_epochs": 3.0,
+  "max_steps": -1,
+  "lr_scheduler_type": "cosine",
+  "lr_scheduler_kwargs": null,
+  "warmup_ratio": 0.05,
+  "warmup_steps": 0,
+  "log_level": "passive",
+  "log_level_replica": "warning",
+  "log_on_each_node": true,
+  "logging_dir": "/nas/shared/kilab/wangyujia/BIO/sft/qwen-metal_ion_binding-08022141/v0-20250802-215822/runs",
+  "logging_strategy": "steps",
+  "logging_first_step": true,
+  "logging_steps": 1,
+  "logging_nan_inf_filter": true,
+  "save_strategy": "steps",
+  "save_steps": 5.0,
+  "save_total_limit": 5,
+  "save_safetensors": true,
+  "save_on_each_node": false,
+  "save_only_model": true,
+  "restore_callback_states_from_checkpoint": false,
+  "no_cuda": false,
+  "use_cpu": false,
+  "use_mps_device": false,
+  "jit_mode_eval": false,
+  "use_ipex": false,
+  "bf16": true,
+  "fp16": false,
+  "fp16_opt_level": "O1",
+  "half_precision_backend": "auto",
+  "bf16_full_eval": false,
+  "fp16_full_eval": false,
+  "tf32": null,
+  "local_rank": 0,
+  "ddp_backend": null,
+  "tpu_num_cores": null,
+  "tpu_metrics_debug": false,
+  "debug": null,
+  "dataloader_drop_last": false,
+  "eval_steps": 5.0,
+  "dataloader_num_workers": 1,
+  "dataloader_prefetch_factor": null,
+  "past_index": -1,
+  "run_name": "construct",
+  "disable_tqdm": null,
+  "label_names": null,
+  "load_best_model_at_end": false,
+  "metric_for_best_model": "loss",
+  "greater_is_better": false,
+  "ignore_data_skip": false,
+  "fsdp": "",
+  "fsdp_min_num_params": 0,
+  "fsdp_config": null,
+  "tp_size": 0,
+  "fsdp_transformer_layer_cls_to_wrap": null,
+  "accelerator_config": {
+    "dispatch_batches": false
+  },
+  "deepspeed": {
+    "fp16": {
+      "enabled": "auto",
+      "loss_scale": 0,
+      "loss_scale_window": 1000,
+      "initial_scale_power": 16,
+      "hysteresis": 2,
+      "min_loss_scale": 1
+    },
+    "bf16": {
+      "enabled": "auto"
+    },
+    "zero_optimization": {
+      "stage": 3,
+      "offload_optimizer": {
+        "device": "none",
+        "pin_memory": true
+      },
+      "offload_param": {
+        "device": "none",
+        "pin_memory": true
+      },
+      "overlap_comm": false,
+      "contiguous_gradients": true,
+      "sub_group_size": 1000000000.0,
+      "reduce_bucket_size": "auto",
+      "zero_quantized_weights": false,
+      "zero_quantized_gradients": false,
+      "stage3_prefetch_bucket_size": "auto",
+      "stage3_param_persistence_threshold": "auto",
+      "stage3_max_live_parameters": 1000000000.0,
+      "stage3_max_reuse_distance": 1000000000.0,
+      "stage3_gather_16bit_weights_on_model_save": true
+    },
+    "gradient_accumulation_steps": "auto",
+    "gradient_clipping": "auto",
+    "steps_per_print": 2000,
+    "train_batch_size": "auto",
+    "train_micro_batch_size_per_gpu": "auto",
+    "wall_clock_breakdown": false
+  },
+  "label_smoothing_factor": 0.0,
+  "optim": "adamw_torch",
+  "optim_args": null,
+  "adafactor": false,
+  "group_by_length": false,
+  "length_column_name": "length",
+  "report_to": [
+    "tensorboard"
+  ],
+  "ddp_find_unused_parameters": null,
+  "ddp_bucket_cap_mb": null,
+  "ddp_broadcast_buffers": null,
+  "dataloader_pin_memory": true,
+  "dataloader_persistent_workers": false,
+  "skip_memory_metrics": true,
+  "use_legacy_prediction_loop": false,
+  "push_to_hub": false,
+  "resume_from_checkpoint": null,
+  "hub_model_id": null,
+  "hub_strategy": "every_save",
+  "hub_private_repo": null,
+  "hub_always_push": false,
+  "gradient_checkpointing": true,
+  "gradient_checkpointing_kwargs": null,
+  "include_inputs_for_metrics": false,
+  "include_for_metrics": [],
+  "eval_do_concat_batches": true,
+  "fp16_backend": "auto",
+  "push_to_hub_model_id": null,
+  "push_to_hub_organization": null,
+  "push_to_hub_token": null,
+  "mp_parameters": "",
+  "auto_find_batch_size": false,
+  "full_determinism": false,
+  "torchdynamo": null,
+  "ray_scope": "last",
+  "ddp_timeout": 1800,
+  "torch_compile": false,
+  "torch_compile_backend": null,
+  "torch_compile_mode": null,
+  "include_tokens_per_second": false,
+  "include_num_input_tokens_seen": false,
+  "neftune_noise_alpha": null,
+  "optim_target_modules": null,
+  "batch_eval_metrics": false,
+  "eval_on_start": false,
+  "use_liger_kernel": false,
+  "eval_use_gather_object": false,
+  "average_tokens_across_devices": false,
+  "sortish_sampler": false,
+  "predict_with_generate": false,
+  "generation_max_length": null,
+  "generation_num_beams": null,
+  "generation_config": null,
+  "check_model": true,
+  "acc_strategy": "token",
+  "train_dataloader_shuffle": true,
+  "metric_warmup_step": 0,
+  "fsdp_num": 1,
+  "acc_steps": 1,
+  "eval_use_evalscope": false,
+  "eval_datasets": [],
+  "eval_limit": null,
+  "eval_datasets_args": null,
+  "eval_generation_config": null,
+  "freeze_parameters": [],
+  "freeze_parameters_ratio": 0.0,
+  "trainable_parameters": [],
+  "freeze_llm": false,
+  "freeze_vit": true,
+  "freeze_aligner": true,
+  "target_modules": [
+    "all-linear"
+  ],
+  "target_regex": null,
+  "modules_to_save": [],
+  "lora_rank": 8,
+  "lora_alpha": 32,
+  "lora_dropout": 0.05,
+  "lora_bias": "none",
+  "lora_dtype": null,
+  "lorap_lr_ratio": null,
+  "use_rslora": false,
+  "use_dora": false,
+  "lora_ga_batch_size": 2,
+  "lora_ga_iters": 2,
+  "lora_ga_max_length": 1024,
+  "lora_ga_direction": "ArB2r",
+  "lora_ga_scale": "stable",
+  "lora_ga_stable_gamma": 16,
+  "init_weights": true,
+  "fourier_n_frequency": 2000,
+  "fourier_scaling": 300.0,
+  "boft_block_size": 4,
+  "boft_block_num": 0,
+  "boft_n_butterfly_factor": 1,
+  "boft_dropout": 0.0,
+  "vera_rank": 256,
+  "vera_projection_prng_key": 0,
+  "vera_dropout": 0.0,
+  "vera_d_initial": 0.1,
+  "adapter_act": "gelu",
+  "adapter_length": 128,
+  "use_galore": false,
+  "galore_target_modules": null,
+  "galore_rank": 128,
+  "galore_update_proj_gap": 50,
+  "galore_scale": 1.0,
+  "galore_proj_type": "std",
+  "galore_optim_per_parameter": false,
+  "galore_with_embedding": false,
+  "galore_quantization": false,
+  "galore_proj_quant": false,
+  "galore_proj_bits": 4,
+  "galore_proj_group_size": 256,
+  "galore_cos_threshold": 0.4,
+  "galore_gamma_proj": 2,
+  "galore_queue_size": 5,
+  "adalora_target_r": 8,
+  "adalora_init_r": 12,
+  "adalora_tinit": 0,
+  "adalora_tfinal": 0,
+  "adalora_deltaT": 1,
+  "adalora_beta1": 0.85,
+  "adalora_beta2": 0.85,
+  "adalora_orth_reg_weight": 0.5,
+  "llamapro_num_new_blocks": 4,
+  "llamapro_num_groups": null,
+  "lisa_activated_layers": 0,
+  "lisa_step_interval": 20,
+  "reft_layer_key": null,
+  "reft_layers": null,
+  "reft_rank": 4,
+  "reft_intervention_type": "LoreftIntervention",
+  "reft_args": null,
+  "swanlab_token": null,
+  "swanlab_project": null,
+  "swanlab_workspace": null,
+  "swanlab_exp_name": null,
+  "swanlab_mode": "cloud",
+  "add_version": true,
+  "resume_only_model": false,
+  "create_checkpoint_symlink": false,
+  "packing": false,
+  "lazy_tokenize": false,
+  "loss_type": null,
+  "optimizer": null,
+  "metric": null,
+  "zero_hpz_partition_size": null,
+  "rank": 0,
+  "global_world_size": 8,
+  "local_world_size": 8,
+  "model_suffix": "checkpoint-50-merged",
+  "model_info": "ModelInfo(model_type='qwen2_5', model_dir='/oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged', torch_dtype=torch.bfloat16, max_model_len=32768, quant_method=None, quant_bits=None, rope_scaling=None, config=None, task_type='causal_lm', num_labels=None)",
+  "model_meta": "ModelMeta(model_type='qwen2_5', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct', hf_model_id='Qwen/Qwen2.5-3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct', hf_model_id='Qwen/Qwen2.5-7B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct', hf_model_id='Qwen/Qwen2.5-14B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct', hf_model_id='Qwen/Qwen2.5-32B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct', hf_model_id='Qwen/Qwen2.5-72B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B', hf_model_id='Qwen/Qwen2.5-0.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B', hf_model_id='Qwen/Qwen2.5-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B', hf_model_id='Qwen/Qwen2.5-3B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B', hf_model_id='Qwen/Qwen2.5-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B', hf_model_id='Qwen/Qwen2.5-14B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B', hf_model_id='Qwen/Qwen2.5-32B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B', hf_model_id='Qwen/Qwen2.5-72B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-3B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-7B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-14B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-32B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-72B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B', hf_model_id='Qwen/Qwen2.5-Coder-0.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B', hf_model_id='Qwen/Qwen2.5-Coder-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B', hf_model_id='Qwen/Qwen2.5-Coder-3B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B', hf_model_id='Qwen/Qwen2.5-Coder-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B', hf_model_id='Qwen/Qwen2.5-Coder-14B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B', hf_model_id='Qwen/Qwen2.5-Coder-32B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=['coding'])], template='qwen2_5', get_function=<function get_model_tokenizer_with_flash_attn at 0x7fbf7497de10>, model_arch='llama', architectures=['Qwen2ForCausalLM'], additional_saved_files=[], torch_dtype=None, is_multimodal=False, is_reward=False, task_type=None, ignore_patterns=[], requires=['transformers>=4.37'], tags=[])",
+  "model_dir": "/oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged",
+  "hub": "<class 'swift.hub.hub.MSHub'>",
+  "evaluation_strategy": "steps",
+  "training_args": "Seq2SeqTrainingArguments(output_dir='/nas/shared/kilab/wangyujia/BIO/sft/qwen-metal_ion_binding-08022141/v0-20250802-215822', overwrite_output_dir=False, do_train=False, do_eval=True, do_predict=False, eval_strategy=<IntervalStrategy.STEPS: 'steps'>, prediction_loss_only=False, per_device_train_batch_size=2, per_device_eval_batch_size=2, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=4, eval_accumulation_steps=None, eval_delay=0, torch_empty_cache_steps=None, learning_rate=1e-05, weight_decay=0.1, adam_beta1=0.9, adam_beta2=0.95, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=3.0, max_steps=-1, lr_scheduler_type=<SchedulerType.COSINE: 'cosine'>, lr_scheduler_kwargs=None, warmup_ratio=0.05, warmup_steps=0, log_level='passive', log_level_replica='warning', log_on_each_node=True, logging_dir='/nas/shared/kilab/wangyujia/BIO/sft/qwen-metal_ion_binding-08022141/v0-20250802-215822/runs', logging_strategy=<IntervalStrategy.STEPS: 'steps'>, logging_first_step=True, logging_steps=1, logging_nan_inf_filter=True, save_strategy=<SaveStrategy.STEPS: 'steps'>, save_steps=5, save_total_limit=5, save_safetensors=True, save_on_each_node=False, save_only_model=True, restore_callback_states_from_checkpoint=False, no_cuda=False, use_cpu=False, use_mps_device=False, seed=42, data_seed=42, jit_mode_eval=False, use_ipex=False, bf16=True, fp16=False, fp16_opt_level='O1', half_precision_backend='auto', bf16_full_eval=False, fp16_full_eval=False, tf32=None, local_rank=0, ddp_backend=None, tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=5, dataloader_num_workers=1, dataloader_prefetch_factor=10, past_index=-1, run_name='/nas/shared/kilab/wangyujia/BIO/sft/qwen-metal_ion_binding-08022141/v0-20250802-215822', disable_tqdm=False, remove_unused_columns=False, label_names=None, load_best_model_at_end=False, metric_for_best_model='loss', greater_is_better=False, ignore_data_skip=False, fsdp=[], fsdp_min_num_params=0, fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, tp_size=0, fsdp_transformer_layer_cls_to_wrap=None, accelerator_config=AcceleratorConfig(split_batches=False, dispatch_batches=False, even_batches=True, use_seedable_sampler=True, non_blocking=False, gradient_accumulation_kwargs=None, use_configured_state=False), deepspeed={'fp16': {'enabled': 'auto', 'loss_scale': 0, 'loss_scale_window': 1000, 'initial_scale_power': 16, 'hysteresis': 2, 'min_loss_scale': 1}, 'bf16': {'enabled': 'auto'}, 'zero_optimization': {'stage': 3, 'offload_optimizer': {'device': 'none', 'pin_memory': True}, 'offload_param': {'device': 'none', 'pin_memory': True}, 'overlap_comm': False, 'contiguous_gradients': True, 'sub_group_size': 1000000000.0, 'reduce_bucket_size': 'auto', 'zero_quantized_weights': False, 'zero_quantized_gradients': False, 'stage3_prefetch_bucket_size': 'auto', 'stage3_param_persistence_threshold': 'auto', 'stage3_max_live_parameters': 1000000000.0, 'stage3_max_reuse_distance': 1000000000.0, 'stage3_gather_16bit_weights_on_model_save': True}, 'gradient_accumulation_steps': 'auto', 'gradient_clipping': 'auto', 'steps_per_print': 2000, 'train_batch_size': 'auto', 'train_micro_batch_size_per_gpu': 'auto', 'wall_clock_breakdown': False}, label_smoothing_factor=0.0, optim=<OptimizerNames.ADAMW_TORCH: 'adamw_torch'>, optim_args=None, adafactor=False, group_by_length=False, length_column_name='length', report_to=['tensorboard'], ddp_find_unused_parameters=None, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, dataloader_pin_memory=True, dataloader_persistent_workers=False, skip_memory_metrics=True, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, hub_model_id=None, hub_strategy=<HubStrategy.EVERY_SAVE: 'every_save'>, hub_token=None, hub_private_repo=None, hub_always_push=False, gradient_checkpointing=True, gradient_checkpointing_kwargs=None, include_inputs_for_metrics=False, include_for_metrics=[], eval_do_concat_batches=True, fp16_backend='auto', push_to_hub_model_id=None, push_to_hub_organization=None, push_to_hub_token=None, mp_parameters='', auto_find_batch_size=False, full_determinism=False, torchdynamo=None, ray_scope='last', ddp_timeout=1800, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, include_tokens_per_second=None, include_num_input_tokens_seen=None, neftune_noise_alpha=None, optim_target_modules=None, batch_eval_metrics=False, eval_on_start=False, use_liger_kernel=False, eval_use_gather_object=False, average_tokens_across_devices=None, sortish_sampler=False, predict_with_generate=False, generation_max_length=None, generation_num_beams=None, generation_config=None, check_model=True, acc_strategy='token', train_dataloader_shuffle=True, metric_warmup_step=0, fsdp_num=1, acc_steps=1, eval_use_evalscope=False, eval_datasets=[], eval_limit=None, eval_datasets_args=None, eval_generation_config=None, train_type='lora', optimizer=None, local_repo_path=None, galore_config=None)"
+}

BioReason_new/wandb/run-20250812_131300-zr3pdtci/logs/debug-internal.log

@@ -0,0 +1,23 @@
+{"time":"2025-08-12T13:13:01.108626404+08:00","level":"INFO","msg":"stream: starting","core version":"0.21.1"}
+{"time":"2025-08-12T13:13:02.160994324+08:00","level":"INFO","msg":"stream: created new stream","id":"zr3pdtci"}
+{"time":"2025-08-12T13:13:02.161037833+08:00","level":"INFO","msg":"stream: started","id":"zr3pdtci"}
+{"time":"2025-08-12T13:13:02.16105997+08:00","level":"INFO","msg":"writer: started","stream_id":"zr3pdtci"}
+{"time":"2025-08-12T13:13:02.161100503+08:00","level":"INFO","msg":"sender: started","stream_id":"zr3pdtci"}
+{"time":"2025-08-12T13:13:02.161076037+08:00","level":"INFO","msg":"handler: started","stream_id":"zr3pdtci"}
+{"time":"2025-08-12T13:13:08.73352003+08:00","level":"INFO","msg":"stream: closing","id":"zr3pdtci"}
+{"time":"2025-08-12T13:13:09.557916985+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/zr3pdtci/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250812%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250812T051309Z&X-Goog-Expires=86399&X-Goog-Signature=138c39feefe589f0780f5c3a2264b19164b3810353e82168fca5f72c77a0ff8ab6814d953abfad7556522255795b5088904fa68d9a4d44be2894692dda633e81dc549c16cca0ead7036ed628003be06ebe81cf3bbaf6f4603fa2b5788863b914e3fdd5fcece8aa1f69a1e301eba60c33d14778c9d55f374f1e63b33428e1bfb9a65cf747a8e4e7ff3c5951ba44402b91691b364dc5c9cd74c9b94b729124a72c3a31f170c9fe00bc2ed1881d46c6f184fbfbe7904a5e3bfd8654a9b8c122d9de9e23dce34c7b9f33015b95e28e2a9830bca505dc80c4dd32b203fbe486a67120ba82245c6231187385d7ba86c1e17dc3b0aabd6c7b4c82017dc9208ef261033c&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.8.118:39586->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/zr3pdtci/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250812%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250812T051309Z&X-Goog-Expires=86399&X-Goog-Signature=138c39feefe589f0780f5c3a2264b19164b3810353e82168fca5f72c77a0ff8ab6814d953abfad7556522255795b5088904fa68d9a4d44be2894692dda633e81dc549c16cca0ead7036ed628003be06ebe81cf3bbaf6f4603fa2b5788863b914e3fdd5fcece8aa1f69a1e301eba60c33d14778c9d55f374f1e63b33428e1bfb9a65cf747a8e4e7ff3c5951ba44402b91691b364dc5c9cd74c9b94b729124a72c3a31f170c9fe00bc2ed1881d46c6f184fbfbe7904a5e3bfd8654a9b8c122d9de9e23dce34c7b9f33015b95e28e2a9830bca505dc80c4dd32b203fbe486a67120ba82245c6231187385d7ba86c1e17dc3b0aabd6c7b4c82017dc9208ef261033c&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-12T13:13:09.571553285+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/zr3pdtci/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250812%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250812T051309Z&X-Goog-Expires=86399&X-Goog-Signature=ba3376ee29fe30c376fb9382ae1e910d7358057b038a08cf2dbbeb15d1b7959f78cde6d4088e7c59a007a66da465739db944f3898a2a6a8056b44b13665594e3c14405e85d0feeb7e2dd1e1c297d3739ead411570cb8f8242e30aa7ac4cc754c91b063ed9f8113701d729238667886da423c5a52c40e49606aede11302d3b922a364dd3873c5565ffd71ef97a53e79bb1ad7d333cce435e1dbb9fdbe8ef87543b5f34348f3c0f50084bba7df83b57ca5e37d45189191f1f86d4b95d89683f89eab18e2f9841aac3a4b9b9763154753c6f889c3f0c7e4aecb36220a6707d7dfbee9b3b240f415e8c83cc6dc23ecc23e3104fb2a2f07a4a075331295550cb42238&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.8.118:39582->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/zr3pdtci/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250812%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250812T051309Z&X-Goog-Expires=86399&X-Goog-Signature=ba3376ee29fe30c376fb9382ae1e910d7358057b038a08cf2dbbeb15d1b7959f78cde6d4088e7c59a007a66da465739db944f3898a2a6a8056b44b13665594e3c14405e85d0feeb7e2dd1e1c297d3739ead411570cb8f8242e30aa7ac4cc754c91b063ed9f8113701d729238667886da423c5a52c40e49606aede11302d3b922a364dd3873c5565ffd71ef97a53e79bb1ad7d333cce435e1dbb9fdbe8ef87543b5f34348f3c0f50084bba7df83b57ca5e37d45189191f1f86d4b95d89683f89eab18e2f9841aac3a4b9b9763154753c6f889c3f0c7e4aecb36220a6707d7dfbee9b3b240f415e8c83cc6dc23ecc23e3104fb2a2f07a4a075331295550cb42238&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-12T13:13:09.661494759+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/zr3pdtci/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250812%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250812T051309Z&X-Goog-Expires=86399&X-Goog-Signature=65d4e0d34ebdaa8bac5b4cd6c135204f00f5907278622ab191037fc2a5cb99a426481eadac62db007021a0d27d982d1f49ccacbe82920d53ef716bba62e271239f454c342305deb0bc880bfd60bb8a55870e869a445ced72496005481e34fa97512fd8a5f00c9faaac098d661c8246bb5ab8cd5537ed25b3f3305dfa297f69b8dcbd61c8cf07e137fbc5b25d7e03f7427d3cbf6ed8c9132b72e3c450a9c1969ca102016ba31a04ac0bcfcb4ee1ccb1af1366bf88a04dd4bdec32c8ca42f09276420822be59548ec2f693b71fd42958b610c6f96d3020c12e1c655ee953073b439b2ace4a590abc4490235ce17ca37a5cd8caa440ed31b894f9b5446737da527f&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.8.118:39570->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/zr3pdtci/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250812%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250812T051309Z&X-Goog-Expires=86399&X-Goog-Signature=65d4e0d34ebdaa8bac5b4cd6c135204f00f5907278622ab191037fc2a5cb99a426481eadac62db007021a0d27d982d1f49ccacbe82920d53ef716bba62e271239f454c342305deb0bc880bfd60bb8a55870e869a445ced72496005481e34fa97512fd8a5f00c9faaac098d661c8246bb5ab8cd5537ed25b3f3305dfa297f69b8dcbd61c8cf07e137fbc5b25d7e03f7427d3cbf6ed8c9132b72e3c450a9c1969ca102016ba31a04ac0bcfcb4ee1ccb1af1366bf88a04dd4bdec32c8ca42f09276420822be59548ec2f693b71fd42958b610c6f96d3020c12e1c655ee953073b439b2ace4a590abc4490235ce17ca37a5cd8caa440ed31b894f9b5446737da527f&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-12T13:15:13.828654367+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/zr3pdtci/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250812%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250812T051303Z&X-Goog-Expires=86399&X-Goog-Signature=61075e983922f3c8156d622bd1220d9e33613bdf5a3a15e70dc02495dd21ed4a27b4878630ad40fc98bbc5e9d97f7cbd6628671dc561fe7b8a04dadd9d5b4d3fd7a90f52e7c5b528e286503ac876ad3c8d84c9517ece6ba2cb4eba83423f4c2c0934da96dfc1631a3db950c65785d04461db81841e71103eea269269dbb2d38c28649f379358827eeb3365b14dfed462f0152614c6c243df9d4bee02ab6c1903138248a07186d58542e829f8bb9f1ae9294b7c1e4138f21066f14f69726847ec0f4b18898a6b8430ee42482beab7654668dc3b39e4ff45b5be12c4f768fcad2b8597c9c78b41c6c666db898849b9c687af7aec0e9f7e554804466aa1d24d953b&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.8.118:42648->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/zr3pdtci/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250812%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250812T051303Z&X-Goog-Expires=86399&X-Goog-Signature=61075e983922f3c8156d622bd1220d9e33613bdf5a3a15e70dc02495dd21ed4a27b4878630ad40fc98bbc5e9d97f7cbd6628671dc561fe7b8a04dadd9d5b4d3fd7a90f52e7c5b528e286503ac876ad3c8d84c9517ece6ba2cb4eba83423f4c2c0934da96dfc1631a3db950c65785d04461db81841e71103eea269269dbb2d38c28649f379358827eeb3365b14dfed462f0152614c6c243df9d4bee02ab6c1903138248a07186d58542e829f8bb9f1ae9294b7c1e4138f21066f14f69726847ec0f4b18898a6b8430ee42482beab7654668dc3b39e4ff45b5be12c4f768fcad2b8597c9c78b41c6c666db898849b9c687af7aec0e9f7e554804466aa1d24d953b&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-12T13:15:13.836170243+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/zr3pdtci/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250812%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250812T051303Z&X-Goog-Expires=86399&X-Goog-Signature=036a136a48cc430b2e124c6a943bd3365d0148bc43f506ac3f83240e62fddef675010a7570e490507404976772f3c801d25c150805086db53f8d97e556c9a7d808333fb6fb906571829baac68a31803efb6406d05de6c960dfb4b34bc7f4a211fdcd35bbb67b78b70cd99028fa098777ec10b296a9152931092b7f023604c9ac1bbe017f8f5d1f2cfd40e62862a63667e27832daa864ce133d6fa234ee8bc848b4eda73609490a4a1131c125fef8a276fa267d984774633ac0be08bad83765243743aada64d391d87a24fc4c2919668044acd67af20bac5bbda64742f0e6d3f0b6f43e0482ffba9aee3beb99ffb8bdb2f6a0587bba67046a76eec719aa3c139b&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.8.118:42646->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/zr3pdtci/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250812%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250812T051303Z&X-Goog-Expires=86399&X-Goog-Signature=036a136a48cc430b2e124c6a943bd3365d0148bc43f506ac3f83240e62fddef675010a7570e490507404976772f3c801d25c150805086db53f8d97e556c9a7d808333fb6fb906571829baac68a31803efb6406d05de6c960dfb4b34bc7f4a211fdcd35bbb67b78b70cd99028fa098777ec10b296a9152931092b7f023604c9ac1bbe017f8f5d1f2cfd40e62862a63667e27832daa864ce133d6fa234ee8bc848b4eda73609490a4a1131c125fef8a276fa267d984774633ac0be08bad83765243743aada64d391d87a24fc4c2919668044acd67af20bac5bbda64742f0e6d3f0b6f43e0482ffba9aee3beb99ffb8bdb2f6a0587bba67046a76eec719aa3c139b&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-12T13:15:16.514141941+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/zr3pdtci/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250812%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250812T051303Z&X-Goog-Expires=86399&X-Goog-Signature=61075e983922f3c8156d622bd1220d9e33613bdf5a3a15e70dc02495dd21ed4a27b4878630ad40fc98bbc5e9d97f7cbd6628671dc561fe7b8a04dadd9d5b4d3fd7a90f52e7c5b528e286503ac876ad3c8d84c9517ece6ba2cb4eba83423f4c2c0934da96dfc1631a3db950c65785d04461db81841e71103eea269269dbb2d38c28649f379358827eeb3365b14dfed462f0152614c6c243df9d4bee02ab6c1903138248a07186d58542e829f8bb9f1ae9294b7c1e4138f21066f14f69726847ec0f4b18898a6b8430ee42482beab7654668dc3b39e4ff45b5be12c4f768fcad2b8597c9c78b41c6c666db898849b9c687af7aec0e9f7e554804466aa1d24d953b&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.8.118:39884->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/zr3pdtci/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250812%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250812T051303Z&X-Goog-Expires=86399&X-Goog-Signature=61075e983922f3c8156d622bd1220d9e33613bdf5a3a15e70dc02495dd21ed4a27b4878630ad40fc98bbc5e9d97f7cbd6628671dc561fe7b8a04dadd9d5b4d3fd7a90f52e7c5b528e286503ac876ad3c8d84c9517ece6ba2cb4eba83423f4c2c0934da96dfc1631a3db950c65785d04461db81841e71103eea269269dbb2d38c28649f379358827eeb3365b14dfed462f0152614c6c243df9d4bee02ab6c1903138248a07186d58542e829f8bb9f1ae9294b7c1e4138f21066f14f69726847ec0f4b18898a6b8430ee42482beab7654668dc3b39e4ff45b5be12c4f768fcad2b8597c9c78b41c6c666db898849b9c687af7aec0e9f7e554804466aa1d24d953b&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-12T13:15:22.024313571+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/zr3pdtci/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250812%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250812T051309Z&X-Goog-Expires=86399&X-Goog-Signature=65d4e0d34ebdaa8bac5b4cd6c135204f00f5907278622ab191037fc2a5cb99a426481eadac62db007021a0d27d982d1f49ccacbe82920d53ef716bba62e271239f454c342305deb0bc880bfd60bb8a55870e869a445ced72496005481e34fa97512fd8a5f00c9faaac098d661c8246bb5ab8cd5537ed25b3f3305dfa297f69b8dcbd61c8cf07e137fbc5b25d7e03f7427d3cbf6ed8c9132b72e3c450a9c1969ca102016ba31a04ac0bcfcb4ee1ccb1af1366bf88a04dd4bdec32c8ca42f09276420822be59548ec2f693b71fd42958b610c6f96d3020c12e1c655ee953073b439b2ace4a590abc4490235ce17ca37a5cd8caa440ed31b894f9b5446737da527f&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.8.118:42662->142.250.217.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/zr3pdtci/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250812%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250812T051309Z&X-Goog-Expires=86399&X-Goog-Signature=65d4e0d34ebdaa8bac5b4cd6c135204f00f5907278622ab191037fc2a5cb99a426481eadac62db007021a0d27d982d1f49ccacbe82920d53ef716bba62e271239f454c342305deb0bc880bfd60bb8a55870e869a445ced72496005481e34fa97512fd8a5f00c9faaac098d661c8246bb5ab8cd5537ed25b3f3305dfa297f69b8dcbd61c8cf07e137fbc5b25d7e03f7427d3cbf6ed8c9132b72e3c450a9c1969ca102016ba31a04ac0bcfcb4ee1ccb1af1366bf88a04dd4bdec32c8ca42f09276420822be59548ec2f693b71fd42958b610c6f96d3020c12e1c655ee953073b439b2ace4a590abc4490235ce17ca37a5cd8caa440ed31b894f9b5446737da527f&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-12T13:17:24.906497736+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/zr3pdtci/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250812%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250812T051303Z&X-Goog-Expires=86399&X-Goog-Signature=036a136a48cc430b2e124c6a943bd3365d0148bc43f506ac3f83240e62fddef675010a7570e490507404976772f3c801d25c150805086db53f8d97e556c9a7d808333fb6fb906571829baac68a31803efb6406d05de6c960dfb4b34bc7f4a211fdcd35bbb67b78b70cd99028fa098777ec10b296a9152931092b7f023604c9ac1bbe017f8f5d1f2cfd40e62862a63667e27832daa864ce133d6fa234ee8bc848b4eda73609490a4a1131c125fef8a276fa267d984774633ac0be08bad83765243743aada64d391d87a24fc4c2919668044acd67af20bac5bbda64742f0e6d3f0b6f43e0482ffba9aee3beb99ffb8bdb2f6a0587bba67046a76eec719aa3c139b&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.8.118:55432->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/zr3pdtci/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250812%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250812T051303Z&X-Goog-Expires=86399&X-Goog-Signature=036a136a48cc430b2e124c6a943bd3365d0148bc43f506ac3f83240e62fddef675010a7570e490507404976772f3c801d25c150805086db53f8d97e556c9a7d808333fb6fb906571829baac68a31803efb6406d05de6c960dfb4b34bc7f4a211fdcd35bbb67b78b70cd99028fa098777ec10b296a9152931092b7f023604c9ac1bbe017f8f5d1f2cfd40e62862a63667e27832daa864ce133d6fa234ee8bc848b4eda73609490a4a1131c125fef8a276fa267d984774633ac0be08bad83765243743aada64d391d87a24fc4c2919668044acd67af20bac5bbda64742f0e6d3f0b6f43e0482ffba9aee3beb99ffb8bdb2f6a0587bba67046a76eec719aa3c139b&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-12T13:17:33.096841516+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/zr3pdtci/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250812%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250812T051309Z&X-Goog-Expires=86399&X-Goog-Signature=138c39feefe589f0780f5c3a2264b19164b3810353e82168fca5f72c77a0ff8ab6814d953abfad7556522255795b5088904fa68d9a4d44be2894692dda633e81dc549c16cca0ead7036ed628003be06ebe81cf3bbaf6f4603fa2b5788863b914e3fdd5fcece8aa1f69a1e301eba60c33d14778c9d55f374f1e63b33428e1bfb9a65cf747a8e4e7ff3c5951ba44402b91691b364dc5c9cd74c9b94b729124a72c3a31f170c9fe00bc2ed1881d46c6f184fbfbe7904a5e3bfd8654a9b8c122d9de9e23dce34c7b9f33015b95e28e2a9830bca505dc80c4dd32b203fbe486a67120ba82245c6231187385d7ba86c1e17dc3b0aabd6c7b4c82017dc9208ef261033c&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.8.118:59370->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/zr3pdtci/config.yaml?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250812%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250812T051309Z&X-Goog-Expires=86399&X-Goog-Signature=138c39feefe589f0780f5c3a2264b19164b3810353e82168fca5f72c77a0ff8ab6814d953abfad7556522255795b5088904fa68d9a4d44be2894692dda633e81dc549c16cca0ead7036ed628003be06ebe81cf3bbaf6f4603fa2b5788863b914e3fdd5fcece8aa1f69a1e301eba60c33d14778c9d55f374f1e63b33428e1bfb9a65cf747a8e4e7ff3c5951ba44402b91691b364dc5c9cd74c9b94b729124a72c3a31f170c9fe00bc2ed1881d46c6f184fbfbe7904a5e3bfd8654a9b8c122d9de9e23dce34c7b9f33015b95e28e2a9830bca505dc80c4dd32b203fbe486a67120ba82245c6231187385d7ba86c1e17dc3b0aabd6c7b4c82017dc9208ef261033c&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-12T13:17:33.124262755+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/zr3pdtci/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250812%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250812T051303Z&X-Goog-Expires=86399&X-Goog-Signature=61075e983922f3c8156d622bd1220d9e33613bdf5a3a15e70dc02495dd21ed4a27b4878630ad40fc98bbc5e9d97f7cbd6628671dc561fe7b8a04dadd9d5b4d3fd7a90f52e7c5b528e286503ac876ad3c8d84c9517ece6ba2cb4eba83423f4c2c0934da96dfc1631a3db950c65785d04461db81841e71103eea269269dbb2d38c28649f379358827eeb3365b14dfed462f0152614c6c243df9d4bee02ab6c1903138248a07186d58542e829f8bb9f1ae9294b7c1e4138f21066f14f69726847ec0f4b18898a6b8430ee42482beab7654668dc3b39e4ff45b5be12c4f768fcad2b8597c9c78b41c6c666db898849b9c687af7aec0e9f7e554804466aa1d24d953b&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.8.118:53630->142.250.73.91:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/zr3pdtci/requirements.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250812%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250812T051303Z&X-Goog-Expires=86399&X-Goog-Signature=61075e983922f3c8156d622bd1220d9e33613bdf5a3a15e70dc02495dd21ed4a27b4878630ad40fc98bbc5e9d97f7cbd6628671dc561fe7b8a04dadd9d5b4d3fd7a90f52e7c5b528e286503ac876ad3c8d84c9517ece6ba2cb4eba83423f4c2c0934da96dfc1631a3db950c65785d04461db81841e71103eea269269dbb2d38c28649f379358827eeb3365b14dfed462f0152614c6c243df9d4bee02ab6c1903138248a07186d58542e829f8bb9f1ae9294b7c1e4138f21066f14f69726847ec0f4b18898a6b8430ee42482beab7654668dc3b39e4ff45b5be12c4f768fcad2b8597c9c78b41c6c666db898849b9c687af7aec0e9f7e554804466aa1d24d953b&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-12T13:17:33.139133433+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/zr3pdtci/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250812%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250812T051309Z&X-Goog-Expires=86399&X-Goog-Signature=ba3376ee29fe30c376fb9382ae1e910d7358057b038a08cf2dbbeb15d1b7959f78cde6d4088e7c59a007a66da465739db944f3898a2a6a8056b44b13665594e3c14405e85d0feeb7e2dd1e1c297d3739ead411570cb8f8242e30aa7ac4cc754c91b063ed9f8113701d729238667886da423c5a52c40e49606aede11302d3b922a364dd3873c5565ffd71ef97a53e79bb1ad7d333cce435e1dbb9fdbe8ef87543b5f34348f3c0f50084bba7df83b57ca5e37d45189191f1f86d4b95d89683f89eab18e2f9841aac3a4b9b9763154753c6f889c3f0c7e4aecb36220a6707d7dfbee9b3b240f415e8c83cc6dc23ecc23e3104fb2a2f07a4a075331295550cb42238&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.8.118:59376->142.250.73.123:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/zr3pdtci/wandb-summary.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250812%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250812T051309Z&X-Goog-Expires=86399&X-Goog-Signature=ba3376ee29fe30c376fb9382ae1e910d7358057b038a08cf2dbbeb15d1b7959f78cde6d4088e7c59a007a66da465739db944f3898a2a6a8056b44b13665594e3c14405e85d0feeb7e2dd1e1c297d3739ead411570cb8f8242e30aa7ac4cc754c91b063ed9f8113701d729238667886da423c5a52c40e49606aede11302d3b922a364dd3873c5565ffd71ef97a53e79bb1ad7d333cce435e1dbb9fdbe8ef87543b5f34348f3c0f50084bba7df83b57ca5e37d45189191f1f86d4b95d89683f89eab18e2f9841aac3a4b9b9763154753c6f889c3f0c7e4aecb36220a6707d7dfbee9b3b240f415e8c83cc6dc23ecc23e3104fb2a2f07a4a075331295550cb42238&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-12T13:17:37.290872326+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/zr3pdtci/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250812%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250812T051309Z&X-Goog-Expires=86399&X-Goog-Signature=65d4e0d34ebdaa8bac5b4cd6c135204f00f5907278622ab191037fc2a5cb99a426481eadac62db007021a0d27d982d1f49ccacbe82920d53ef716bba62e271239f454c342305deb0bc880bfd60bb8a55870e869a445ced72496005481e34fa97512fd8a5f00c9faaac098d661c8246bb5ab8cd5537ed25b3f3305dfa297f69b8dcbd61c8cf07e137fbc5b25d7e03f7427d3cbf6ed8c9132b72e3c450a9c1969ca102016ba31a04ac0bcfcb4ee1ccb1af1366bf88a04dd4bdec32c8ca42f09276420822be59548ec2f693b71fd42958b610c6f96d3020c12e1c655ee953073b439b2ace4a590abc4490235ce17ca37a5cd8caa440ed31b894f9b5446737da527f&X-Goog-SignedHeaders=host&X-User=gia0603yucca\": read tcp 10.1.8.118:39894->142.250.73.155:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-production.appspot.com/gia0603yucca/protein-llm-contrastive/zr3pdtci/output.log?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250812%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250812T051309Z&X-Goog-Expires=86399&X-Goog-Signature=65d4e0d34ebdaa8bac5b4cd6c135204f00f5907278622ab191037fc2a5cb99a426481eadac62db007021a0d27d982d1f49ccacbe82920d53ef716bba62e271239f454c342305deb0bc880bfd60bb8a55870e869a445ced72496005481e34fa97512fd8a5f00c9faaac098d661c8246bb5ab8cd5537ed25b3f3305dfa297f69b8dcbd61c8cf07e137fbc5b25d7e03f7427d3cbf6ed8c9132b72e3c450a9c1969ca102016ba31a04ac0bcfcb4ee1ccb1af1366bf88a04dd4bdec32c8ca42f09276420822be59548ec2f693b71fd42958b610c6f96d3020c12e1c655ee953073b439b2ace4a590abc4490235ce17ca37a5cd8caa440ed31b894f9b5446737da527f&X-Goog-SignedHeaders=host&X-User=gia0603yucca"}
+{"time":"2025-08-12T13:17:47.6529587+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-08-12T13:17:48.610451999+08:00","level":"INFO","msg":"handler: closed","stream_id":"zr3pdtci"}
+{"time":"2025-08-12T13:17:48.613740545+08:00","level":"INFO","msg":"sender: closed","stream_id":"zr3pdtci"}
+{"time":"2025-08-12T13:17:48.613750169+08:00","level":"INFO","msg":"stream: closed","id":"zr3pdtci"}

BioReason_new/wandb/run-20250812_131300-zr3pdtci/logs/debug.log

@@ -0,0 +1,22 @@
+2025-08-12 13:13:00,887 INFO    MainThread:87959 [wandb_setup.py:_flush():80] Current SDK version is 0.21.1
+2025-08-12 13:13:00,887 INFO    MainThread:87959 [wandb_setup.py:_flush():80] Configure stats pid to 87959
+2025-08-12 13:13:00,888 INFO    MainThread:87959 [wandb_setup.py:_flush():80] Loading settings from /root/.config/wandb/settings
+2025-08-12 13:13:00,888 INFO    MainThread:87959 [wandb_setup.py:_flush():80] Loading settings from /nas/shared/kilab/wangyujia/BioReason_new/wandb/settings
+2025-08-12 13:13:00,888 INFO    MainThread:87959 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-08-12 13:13:00,888 INFO    MainThread:87959 [wandb_init.py:setup_run_log_directory():703] Logging user logs to /nas/shared/kilab/wangyujia/BioReason_new/wandb/run-20250812_131300-zr3pdtci/logs/debug.log
+2025-08-12 13:13:00,888 INFO    MainThread:87959 [wandb_init.py:setup_run_log_directory():704] Logging internal logs to /nas/shared/kilab/wangyujia/BioReason_new/wandb/run-20250812_131300-zr3pdtci/logs/debug-internal.log
+2025-08-12 13:13:00,888 INFO    MainThread:87959 [wandb_init.py:init():830] calling init triggers
+2025-08-12 13:13:00,888 INFO    MainThread:87959 [wandb_init.py:init():835] wandb.init called with sweep_config: {}
+config: {'text_model_name': '/oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged', 'protein_model_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m', 'qformer_model_name': '/nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft', 'cache_dir': '/model-weights', 'num_query_tokens': 8, 'train_dataset': '/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl', 'valid_dataset': '/nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl', 'eval_dataset': True, 'output_dir': './contrastive_outputs', 'num_epochs': 10, 'batch_size': 32, 'learning_rate': 0.0001, 'weight_decay': 0.01, 'warmup_steps': 1000, 'gradient_accumulation_steps': 1, 'temperature': 0.07, 'freeze_protein_model': True, 'freeze_text_model': True, 'protein_weight': 1.0, 'text_weight': 1.0, 'enable_ptm': True, 'ptm_weight': 1.0, 'max_length_protein': 1024, 'max_length_text': 512, 'num_workers': 8, 'logging_steps': 100, 'eval_steps': 500, 'save_steps': 1000, 'save_total_limit': 3, 'fp16': False, 'bf16': False, 'seed': 42, 'use_wandb': True, 'wandb_project': 'protein-llm-contrastive', 'wandb_entity': None, '_wandb': {}}
+2025-08-12 13:13:00,888 INFO    MainThread:87959 [wandb_init.py:init():871] starting backend
+2025-08-12 13:13:01,096 INFO    MainThread:87959 [wandb_init.py:init():874] sending inform_init request
+2025-08-12 13:13:01,103 INFO    MainThread:87959 [wandb_init.py:init():882] backend started and connected
+2025-08-12 13:13:01,104 INFO    MainThread:87959 [wandb_init.py:init():953] updated telemetry
+2025-08-12 13:13:01,135 INFO    MainThread:87959 [wandb_init.py:init():977] communicating run to backend with 90.0 second timeout
+2025-08-12 13:13:02,774 INFO    MainThread:87959 [wandb_init.py:init():1029] starting run threads in backend
+2025-08-12 13:13:02,883 INFO    MainThread:87959 [wandb_run.py:_console_start():2494] atexit reg
+2025-08-12 13:13:02,883 INFO    MainThread:87959 [wandb_run.py:_redirect():2342] redirect: wrap_raw
+2025-08-12 13:13:02,883 INFO    MainThread:87959 [wandb_run.py:_redirect():2411] Wrapping output streams.
+2025-08-12 13:13:02,883 INFO    MainThread:87959 [wandb_run.py:_redirect():2434] Redirects installed.
+2025-08-12 13:13:02,886 INFO    MainThread:87959 [wandb_init.py:init():1075] run started, returning control to user process
+2025-08-12 13:13:08,732 INFO    MsgRouterThr:87959 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles.

BioReason_new/wandb/run-20250812_132050-2cfwmlj6/files/config.yaml

@@ -0,0 +1,200 @@
+_wandb:
+    value:
+        cli_version: 0.21.1
+        e:
+            crh90aob5eyx2qklfrtgkofnoya37sbu:
+                args:
+                    - --text_model_name
+                    - /oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged
+                    - --protein_model_name
+                    - /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m
+                    - --qformer_model_name
+                    - /nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft
+                    - --num_query_tokens
+                    - "8"
+                    - --train_dataset
+                    - /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl
+                    - --valid_dataset
+                    - /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl
+                    - --output_dir
+                    - ./contrastive_outputs
+                    - --num_epochs
+                    - "10"
+                    - --batch_size
+                    - "32"
+                    - --learning_rate
+                    - "1e-4"
+                    - --temperature
+                    - "0.07"
+                    - --freeze_protein_model
+                    - --freeze_text_model
+                    - --enable_ptm
+                    - --max_length_protein
+                    - "1024"
+                    - --max_length_text
+                    - "512"
+                    - --num_workers
+                    - "8"
+                    - --eval_dataset
+                    - --use_wandb
+                    - --wandb_project
+                    - protein-llm-contrastive
+                    - --logging_steps
+                    - "100"
+                    - --eval_steps
+                    - "500"
+                    - --save_steps
+                    - "1000"
+                codePath: wangyujia/BioReason_new/train_contrastive.py
+                codePathLocal: train_contrastive.py
+                cpu_count: 64
+                cpu_count_logical: 64
+                cudaVersion: "12.1"
+                disk:
+                    /:
+                        total: "1623302262784"
+                        used: "29131816960"
+                email: gia0603yucca@gmail.com
+                executable: /root/miniconda3/envs/bioreason/bin/python
+                git:
+                    commit: b8caf406aa1699c788f0ca6e44a1769452c317db
+                    remote: https://github.com/PorUna-byte/PAR.git
+                gpu: NVIDIA A800-SXM4-80GB
+                gpu_count: 8
+                gpu_nvidia:
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-71607f78-ad31-1ea4-19c1-908e3e31aaf1
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-92b7dbbd-7ef5-3c5f-ce1c-1d179d7fa587
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-bbc35439-ad79-578b-381b-aba6f0cc0168
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-e492e147-ca2e-76f2-85da-4e08e4deeb14
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-8c4f8e67-4b52-5107-3095-0f007e6378ac
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-7063f0b9-4ca2-6a72-522d-1262899ac5ad
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-3b6e9a37-bcf3-387c-7874-4f8de4abd115
+                    - architecture: Ampere
+                      name: NVIDIA A800-SXM4-80GB
+                      uuid: GPU-92456839-e814-7be9-6817-f3e8da8aa80c
+                host: dsw-265304-f8bc5ff76-4mdt5
+                memory:
+                    total: "549755813888"
+                os: Linux-5.10.134-008.18.kangaroo.al8.x86_64-x86_64-with-glibc2.35
+                program: /nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py
+                python: CPython 3.11.0
+                root: /nas/shared/kilab/wangyujia/BioReason_new
+                startedAt: "2025-08-12T05:20:50.783496Z"
+                writerId: crh90aob5eyx2qklfrtgkofnoya37sbu
+        m: []
+        python_version: 3.11.0
+        t:
+            "1":
+                - 1
+                - 9
+                - 11
+                - 41
+                - 49
+                - 51
+                - 71
+                - 84
+                - 98
+                - 103
+            "2":
+                - 1
+                - 9
+                - 11
+                - 41
+                - 49
+                - 51
+                - 71
+                - 84
+                - 98
+                - 103
+            "3":
+                - 13
+                - 16
+            "4": 3.11.0
+            "5": 0.21.1
+            "6": 4.55.0
+            "12": 0.21.1
+            "13": linux-x86_64
+batch_size:
+    value: 32
+bf16:
+    value: false
+cache_dir:
+    value: /model-weights
+enable_ptm:
+    value: true
+eval_dataset:
+    value: true
+eval_steps:
+    value: 500
+fp16:
+    value: false
+freeze_protein_model:
+    value: true
+freeze_text_model:
+    value: true
+gradient_accumulation_steps:
+    value: 1
+learning_rate:
+    value: 0.0001
+logging_steps:
+    value: 100
+max_length_protein:
+    value: 1024
+max_length_text:
+    value: 512
+num_epochs:
+    value: 10
+num_query_tokens:
+    value: 8
+num_workers:
+    value: 8
+output_dir:
+    value: ./contrastive_outputs
+protein_model_name:
+    value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m
+protein_weight:
+    value: 1
+ptm_weight:
+    value: 1
+qformer_model_name:
+    value: /nas/shared/kilab/wangyujia/ProtT3/plm_model/microsoft
+save_steps:
+    value: 1000
+save_total_limit:
+    value: 3
+seed:
+    value: 42
+temperature:
+    value: 0.07
+text_model_name:
+    value: /oss/wangyujia/BIO/construction_finetuning/alpaca/v1-20250609-141541/checkpoint-50-merged
+text_weight:
+    value: 1
+train_dataset:
+    value: /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/train_set.jsonl
+use_wandb:
+    value: true
+valid_dataset:
+    value: /nas/shared/kilab/wangyujia/ProtT3/data/SwissProtV3/valid_set.jsonl
+wandb_entity:
+    value: null
+wandb_project:
+    value: protein-llm-contrastive
+warmup_steps:
+    value: 1000
+weight_decay:
+    value: 0.01

BioReason_new/wandb/run-20250812_132050-2cfwmlj6/files/output.log

@@ -0,0 +1,16 @@
+Loading model...
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████| 4/4 [00:01<00:00,  2.57it/s]
+Some weights of EsmModel were not initialized from the model checkpoint at /nas/shared/kilab/wangyujia/ProtT3/plm_model/esm2-150m and are newly initialized: ['pooler.dense.bias', 'pooler.dense.weight']
+You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
+Loading datasets...
+Training dataset size: 430595
+Eval dataset size: 10000
+bioreason.trainer.contrast_trainer_new
+Traceback (most recent call last):
+  File "/nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py", line 552, in <module>
+    trainer = main(args)
+              ^^^^^^^^^^
+  File "/nas/shared/kilab/wangyujia/BioReason_new/train_contrastive.py", line 345, in main
+    training_args = ContrastiveTrainingArguments(
+                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+TypeError: ContrastiveTrainingArguments.__init__() got an unexpected keyword argument 'evaluation_strategy'

LAVIS-main/lavis/common/annotator/uniformer/mmseg/models/utils/up_conv_block.py

@@ -0,0 +1,101 @@
+import torch
+import torch.nn as nn
+from annotator.uniformer.mmcv.cnn import ConvModule, build_upsample_layer
+
+
+class UpConvBlock(nn.Module):
+    """Upsample convolution block in decoder for UNet.
+
+    This upsample convolution block consists of one upsample module
+    followed by one convolution block. The upsample module expands the
+    high-level low-resolution feature map and the convolution block fuses
+    the upsampled high-level low-resolution feature map and the low-level
+    high-resolution feature map from encoder.
+
+    Args:
+        conv_block (nn.Sequential): Sequential of convolutional layers.
+        in_channels (int): Number of input channels of the high-level
+        skip_channels (int): Number of input channels of the low-level
+        high-resolution feature map from encoder.
+        out_channels (int): Number of output channels.
+        num_convs (int): Number of convolutional layers in the conv_block.
+            Default: 2.
+        stride (int): Stride of convolutional layer in conv_block. Default: 1.
+        dilation (int): Dilation rate of convolutional layer in conv_block.
+            Default: 1.
+        with_cp (bool): Use checkpoint or not. Using checkpoint will save some
+            memory while slowing down the training speed. Default: False.
+        conv_cfg (dict | None): Config dict for convolution layer.
+            Default: None.
+        norm_cfg (dict | None): Config dict for normalization layer.
+            Default: dict(type='BN').
+        act_cfg (dict | None): Config dict for activation layer in ConvModule.
+            Default: dict(type='ReLU').
+        upsample_cfg (dict): The upsample config of the upsample module in
+            decoder. Default: dict(type='InterpConv'). If the size of
+            high-level feature map is the same as that of skip feature map
+            (low-level feature map from encoder), it does not need upsample the
+            high-level feature map and the upsample_cfg is None.
+        dcn (bool): Use deformable convolution in convolutional layer or not.
+            Default: None.
+        plugins (dict): plugins for convolutional layers. Default: None.
+    """
+
+    def __init__(self,
+                 conv_block,
+                 in_channels,
+                 skip_channels,
+                 out_channels,
+                 num_convs=2,
+                 stride=1,
+                 dilation=1,
+                 with_cp=False,
+                 conv_cfg=None,
+                 norm_cfg=dict(type='BN'),
+                 act_cfg=dict(type='ReLU'),
+                 upsample_cfg=dict(type='InterpConv'),
+                 dcn=None,
+                 plugins=None):
+        super(UpConvBlock, self).__init__()
+        assert dcn is None, 'Not implemented yet.'
+        assert plugins is None, 'Not implemented yet.'
+
+        self.conv_block = conv_block(
+            in_channels=2 * skip_channels,
+            out_channels=out_channels,
+            num_convs=num_convs,
+            stride=stride,
+            dilation=dilation,
+            with_cp=with_cp,
+            conv_cfg=conv_cfg,
+            norm_cfg=norm_cfg,
+            act_cfg=act_cfg,
+            dcn=None,
+            plugins=None)
+        if upsample_cfg is not None:
+            self.upsample = build_upsample_layer(
+                cfg=upsample_cfg,
+                in_channels=in_channels,
+                out_channels=skip_channels,
+                with_cp=with_cp,
+                norm_cfg=norm_cfg,
+                act_cfg=act_cfg)
+        else:
+            self.upsample = ConvModule(
+                in_channels,
+                skip_channels,
+                kernel_size=1,
+                stride=1,
+                padding=0,
+                conv_cfg=conv_cfg,
+                norm_cfg=norm_cfg,
+                act_cfg=act_cfg)
+
+    def forward(self, skip, x):
+        """Forward function."""
+
+        x = self.upsample(x)
+        out = torch.cat([skip, x], dim=1)
+        out = self.conv_block(out)
+
+        return out

LAVIS-main/lavis/common/annotator/uniformer/mmseg/models/utils/weight_init.py

@@ -0,0 +1,62 @@
+"""Modified from https://github.com/rwightman/pytorch-image-
+models/blob/master/timm/models/layers/drop.py."""
+
+import math
+import warnings
+
+import torch
+
+
+def _no_grad_trunc_normal_(tensor, mean, std, a, b):
+    """Reference: https://people.sc.fsu.edu/~jburkardt/presentations
+    /truncated_normal.pdf"""
+
+    def norm_cdf(x):
+        # Computes standard normal cumulative distribution function
+        return (1. + math.erf(x / math.sqrt(2.))) / 2.
+
+    if (mean < a - 2 * std) or (mean > b + 2 * std):
+        warnings.warn(
+            'mean is more than 2 std from [a, b] in nn.init.trunc_normal_. '
+            'The distribution of values may be incorrect.',
+            stacklevel=2)
+
+    with torch.no_grad():
+        # Values are generated by using a truncated uniform distribution and
+        # then using the inverse CDF for the normal distribution.
+        # Get upper and lower cdf values
+        lower_bound = norm_cdf((a - mean) / std)
+        upper_bound = norm_cdf((b - mean) / std)
+
+        # Uniformly fill tensor with values from [l, u], then translate to
+        # [2l-1, 2u-1].
+        tensor.uniform_(2 * lower_bound - 1, 2 * upper_bound - 1)
+
+        # Use inverse cdf transform for normal distribution to get truncated
+        # standard normal
+        tensor.erfinv_()
+
+        # Transform to proper mean, std
+        tensor.mul_(std * math.sqrt(2.))
+        tensor.add_(mean)
+
+        # Clamp to ensure it's in the proper range
+        tensor.clamp_(min=a, max=b)
+        return tensor
+
+
+def trunc_normal_(tensor, mean=0., std=1., a=-2., b=2.):
+    r"""Fills the input Tensor with values drawn from a truncated
+    normal distribution. The values are effectively drawn from the
+    normal distribution :math:`\mathcal{N}(\text{mean}, \text{std}^2)`
+    with values outside :math:`[a, b]` redrawn until they are within
+    the bounds. The method used for generating the random values works
+    best when :math:`a \leq \text{mean} \leq b`.
+    Args:
+        tensor (``torch.Tensor``): an n-dimensional `torch.Tensor`
+        mean (float): the mean of the normal distribution
+        std (float): the standard deviation of the normal distribution
+        a (float): the minimum cutoff value
+        b (float): the maximum cutoff value
+    """
+    return _no_grad_trunc_normal_(tensor, mean, std, a, b)

LAVIS-main/lavis/common/annotator/uniformer/mmseg/ops/__init__.py

@@ -0,0 +1,4 @@
+from .encoding import Encoding
+from .wrappers import Upsample, resize
+
+__all__ = ['Upsample', 'resize', 'Encoding']

LAVIS-main/lavis/common/annotator/uniformer/mmseg/ops/encoding.py

@@ -0,0 +1,74 @@
+import torch
+from torch import nn
+from torch.nn import functional as F
+
+
+class Encoding(nn.Module):
+    """Encoding Layer: a learnable residual encoder.
+
+    Input is of shape  (batch_size, channels, height, width).
+    Output is of shape (batch_size, num_codes, channels).
+
+    Args:
+        channels: dimension of the features or feature channels
+        num_codes: number of code words
+    """
+
+    def __init__(self, channels, num_codes):
+        super(Encoding, self).__init__()
+        # init codewords and smoothing factor
+        self.channels, self.num_codes = channels, num_codes
+        std = 1. / ((num_codes * channels)**0.5)
+        # [num_codes, channels]
+        self.codewords = nn.Parameter(
+            torch.empty(num_codes, channels,
+                        dtype=torch.float).uniform_(-std, std),
+            requires_grad=True)
+        # [num_codes]
+        self.scale = nn.Parameter(
+            torch.empty(num_codes, dtype=torch.float).uniform_(-1, 0),
+            requires_grad=True)
+
+    @staticmethod
+    def scaled_l2(x, codewords, scale):
+        num_codes, channels = codewords.size()
+        batch_size = x.size(0)
+        reshaped_scale = scale.view((1, 1, num_codes))
+        expanded_x = x.unsqueeze(2).expand(
+            (batch_size, x.size(1), num_codes, channels))
+        reshaped_codewords = codewords.view((1, 1, num_codes, channels))
+
+        scaled_l2_norm = reshaped_scale * (
+            expanded_x - reshaped_codewords).pow(2).sum(dim=3)
+        return scaled_l2_norm
+
+    @staticmethod
+    def aggregate(assignment_weights, x, codewords):
+        num_codes, channels = codewords.size()
+        reshaped_codewords = codewords.view((1, 1, num_codes, channels))
+        batch_size = x.size(0)
+
+        expanded_x = x.unsqueeze(2).expand(
+            (batch_size, x.size(1), num_codes, channels))
+        encoded_feat = (assignment_weights.unsqueeze(3) *
+                        (expanded_x - reshaped_codewords)).sum(dim=1)
+        return encoded_feat
+
+    def forward(self, x):
+        assert x.dim() == 4 and x.size(1) == self.channels
+        # [batch_size, channels, height, width]
+        batch_size = x.size(0)
+        # [batch_size, height x width, channels]
+        x = x.view(batch_size, self.channels, -1).transpose(1, 2).contiguous()
+        # assignment_weights: [batch_size, channels, num_codes]
+        assignment_weights = F.softmax(
+            self.scaled_l2(x, self.codewords, self.scale), dim=2)
+        # aggregate
+        encoded_feat = self.aggregate(assignment_weights, x, self.codewords)
+        return encoded_feat
+
+    def __repr__(self):
+        repr_str = self.__class__.__name__
+        repr_str += f'(Nx{self.channels}xHxW =>Nx{self.num_codes}' \
+                    f'x{self.channels})'
+        return repr_str

LAVIS-main/lavis/common/annotator/uniformer/mmseg/ops/wrappers.py

@@ -0,0 +1,50 @@
+import warnings
+
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+def resize(input,
+           size=None,
+           scale_factor=None,
+           mode='nearest',
+           align_corners=None,
+           warning=True):
+    if warning:
+        if size is not None and align_corners:
+            input_h, input_w = tuple(int(x) for x in input.shape[2:])
+            output_h, output_w = tuple(int(x) for x in size)
+            if output_h > input_h or output_w > output_h:
+                if ((output_h > 1 and output_w > 1 and input_h > 1
+                     and input_w > 1) and (output_h - 1) % (input_h - 1)
+                        and (output_w - 1) % (input_w - 1)):
+                    warnings.warn(
+                        f'When align_corners={align_corners}, '
+                        'the output would more aligned if '
+                        f'input size {(input_h, input_w)} is `x+1` and '
+                        f'out size {(output_h, output_w)} is `nx+1`')
+    return F.interpolate(input, size, scale_factor, mode, align_corners)
+
+
+class Upsample(nn.Module):
+
+    def __init__(self,
+                 size=None,
+                 scale_factor=None,
+                 mode='nearest',
+                 align_corners=None):
+        super(Upsample, self).__init__()
+        self.size = size
+        if isinstance(scale_factor, tuple):
+            self.scale_factor = tuple(float(factor) for factor in scale_factor)
+        else:
+            self.scale_factor = float(scale_factor) if scale_factor else None
+        self.mode = mode
+        self.align_corners = align_corners
+
+    def forward(self, x):
+        if not self.size:
+            size = [int(t * self.scale_factor) for t in x.shape[-2:]]
+        else:
+            size = self.size
+        return resize(x, size, None, self.mode, self.align_corners)

LAVIS-main/lavis/common/annotator/uniformer/mmseg/utils/__init__.py

@@ -0,0 +1,4 @@
+from .collect_env import collect_env
+from .logger import get_root_logger
+
+__all__ = ['get_root_logger', 'collect_env']

LAVIS-main/lavis/common/annotator/uniformer/mmseg/utils/collect_env.py

@@ -0,0 +1,17 @@
+from annotator.uniformer.mmcv.utils import collect_env as collect_base_env
+from annotator.uniformer.mmcv.utils import get_git_hash
+
+import annotator.uniformer.mmseg as mmseg
+
+
+def collect_env():
+    """Collect the information of the running environments."""
+    env_info = collect_base_env()
+    env_info['MMSegmentation'] = f'{mmseg.__version__}+{get_git_hash()[:7]}'
+
+    return env_info
+
+
+if __name__ == '__main__':
+    for name, val in collect_env().items():
+        print('{}: {}'.format(name, val))

LAVIS-main/lavis/common/annotator/uniformer/mmseg/utils/logger.py

@@ -0,0 +1,27 @@
+import logging
+
+from annotator.uniformer.mmcv.utils import get_logger
+
+
+def get_root_logger(log_file=None, log_level=logging.INFO):
+    """Get the root logger.
+
+    The logger will be initialized if it has not been initialized. By default a
+    StreamHandler will be added. If `log_file` is specified, a FileHandler will
+    also be added. The name of the root logger is the top-level package name,
+    e.g., "mmseg".
+
+    Args:
+        log_file (str | None): The log filename. If specified, a FileHandler
+            will be added to the root logger.
+        log_level (int): The root logger level. Note that only the process of
+            rank 0 is affected, while other processes will set the level to
+            "Error" and be silent most of the time.
+
+    Returns:
+        logging.Logger: The root logger.
+    """
+
+    logger = get_logger(name='mmseg', log_file=log_file, log_level=log_level)
+
+    return logger

LAVIS-main/lavis/common/vqa_tools/__init__.py

@@ -0,0 +1,8 @@
+"""
+ Copyright (c) 2022, salesforce.com, inc.
+ All rights reserved.
+ SPDX-License-Identifier: BSD-3-Clause
+ For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+"""
+
+__author__ = "aagrawal"

LAVIS-main/lavis/common/vqa_tools/vqa.py

@@ -0,0 +1,211 @@
+"""
+ Copyright (c) 2022, salesforce.com, inc.
+ All rights reserved.
+ SPDX-License-Identifier: BSD-3-Clause
+ For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+"""
+
+__author__ = "aagrawal"
+__version__ = "0.9"
+
+# Interface for accessing the VQA dataset.
+
+# This code is based on the code written by Tsung-Yi Lin for MSCOCO Python API available at the following link:
+# (https://github.com/pdollar/coco/blob/master/PythonAPI/pycocotools/coco.py).
+
+# The following functions are defined:
+#  VQA        - VQA class that loads VQA annotation file and prepares data structures.
+#  getQuesIds - Get question ids that satisfy given filter conditions.
+#  getImgIds  - Get image ids that satisfy given filter conditions.
+#  loadQA     - Load questions and answers with the specified question ids.
+#  showQA     - Display the specified questions and answers.
+#  loadRes    - Load result file and create result object.
+
+# Help on each function can be accessed by: "help(COCO.function)"
+
+import json
+import datetime
+import copy
+
+
+class VQA:
+    def __init__(self, annotation_file=None, question_file=None):
+        """
+        Constructor of VQA helper class for reading and visualizing questions and answers.
+        :param annotation_file (str): location of VQA annotation file
+        :return:
+        """
+        # load dataset
+        self.dataset = {}
+        self.questions = {}
+        self.qa = {}
+        self.qqa = {}
+        self.imgToQA = {}
+        if not annotation_file == None and not question_file == None:
+            print("loading VQA annotations and questions into memory...")
+            time_t = datetime.datetime.utcnow()
+            dataset = json.load(open(annotation_file, "r"))
+            questions = json.load(open(question_file, "r"))
+            self.dataset = dataset
+            self.questions = questions
+            self.createIndex()
+
+    def createIndex(self):
+        # create index
+        print("creating index...")
+        imgToQA = {ann["image_id"]: [] for ann in self.dataset["annotations"]}
+        qa = {ann["question_id"]: [] for ann in self.dataset["annotations"]}
+        qqa = {ann["question_id"]: [] for ann in self.dataset["annotations"]}
+        for ann in self.dataset["annotations"]:
+            imgToQA[ann["image_id"]] += [ann]
+            qa[ann["question_id"]] = ann
+        for ques in self.questions["questions"]:
+            qqa[ques["question_id"]] = ques
+        print("index created!")
+
+        # create class members
+        self.qa = qa
+        self.qqa = qqa
+        self.imgToQA = imgToQA
+
+    def info(self):
+        """
+        Print information about the VQA annotation file.
+        :return:
+        """
+        for key, value in self.datset["info"].items():
+            print("%s: %s" % (key, value))
+
+    def getQuesIds(self, imgIds=[], quesTypes=[], ansTypes=[]):
+        """
+        Get question ids that satisfy given filter conditions. default skips that filter
+        :param 	imgIds    (int array)   : get question ids for given imgs
+                        quesTypes (str array)   : get question ids for given question types
+                        ansTypes  (str array)   : get question ids for given answer types
+        :return:    ids   (int array)   : integer array of question ids
+        """
+        imgIds = imgIds if type(imgIds) == list else [imgIds]
+        quesTypes = quesTypes if type(quesTypes) == list else [quesTypes]
+        ansTypes = ansTypes if type(ansTypes) == list else [ansTypes]
+
+        if len(imgIds) == len(quesTypes) == len(ansTypes) == 0:
+            anns = self.dataset["annotations"]
+        else:
+            if not len(imgIds) == 0:
+                anns = sum(
+                    [self.imgToQA[imgId] for imgId in imgIds if imgId in self.imgToQA],
+                    [],
+                )
+            else:
+                anns = self.dataset["annotations"]
+            anns = (
+                anns
+                if len(quesTypes) == 0
+                else [ann for ann in anns if ann["question_type"] in quesTypes]
+            )
+            anns = (
+                anns
+                if len(ansTypes) == 0
+                else [ann for ann in anns if ann["answer_type"] in ansTypes]
+            )
+        ids = [ann["question_id"] for ann in anns]
+        return ids
+
+    def getImgIds(self, quesIds=[], quesTypes=[], ansTypes=[]):
+        """
+         Get image ids that satisfy given filter conditions. default skips that filter
+         :param quesIds   (int array)   : get image ids for given question ids
+        quesTypes (str array)   : get image ids for given question types
+        ansTypes  (str array)   : get image ids for given answer types
+         :return: ids     (int array)   : integer array of image ids
+        """
+        quesIds = quesIds if type(quesIds) == list else [quesIds]
+        quesTypes = quesTypes if type(quesTypes) == list else [quesTypes]
+        ansTypes = ansTypes if type(ansTypes) == list else [ansTypes]
+
+        if len(quesIds) == len(quesTypes) == len(ansTypes) == 0:
+            anns = self.dataset["annotations"]
+        else:
+            if not len(quesIds) == 0:
+                anns = sum(
+                    [self.qa[quesId] for quesId in quesIds if quesId in self.qa], []
+                )
+            else:
+                anns = self.dataset["annotations"]
+            anns = (
+                anns
+                if len(quesTypes) == 0
+                else [ann for ann in anns if ann["question_type"] in quesTypes]
+            )
+            anns = (
+                anns
+                if len(ansTypes) == 0
+                else [ann for ann in anns if ann["answer_type"] in ansTypes]
+            )
+        ids = [ann["image_id"] for ann in anns]
+        return ids
+
+    def loadQA(self, ids=[]):
+        """
+        Load questions and answers with the specified question ids.
+        :param ids (int array)       : integer ids specifying question ids
+        :return: qa (object array)   : loaded qa objects
+        """
+        if type(ids) == list:
+            return [self.qa[id] for id in ids]
+        elif type(ids) == int:
+            return [self.qa[ids]]
+
+    def showQA(self, anns):
+        """
+        Display the specified annotations.
+        :param anns (array of object): annotations to display
+        :return: None
+        """
+        if len(anns) == 0:
+            return 0
+        for ann in anns:
+            quesId = ann["question_id"]
+            print("Question: %s" % (self.qqa[quesId]["question"]))
+            for ans in ann["answers"]:
+                print("Answer %d: %s" % (ans["answer_id"], ans["answer"]))
+
+    def loadRes(self, resFile, quesFile):
+        """
+        Load result file and return a result object.
+        :param   resFile (str)     : file name of result file
+        :return: res (obj)         : result api object
+        """
+        res = VQA()
+        res.questions = json.load(open(quesFile))
+        res.dataset["info"] = copy.deepcopy(self.questions["info"])
+        res.dataset["task_type"] = copy.deepcopy(self.questions["task_type"])
+        res.dataset["data_type"] = copy.deepcopy(self.questions["data_type"])
+        res.dataset["data_subtype"] = copy.deepcopy(self.questions["data_subtype"])
+        res.dataset["license"] = copy.deepcopy(self.questions["license"])
+
+        print("Loading and preparing results...     ")
+        time_t = datetime.datetime.utcnow()
+        anns = json.load(open(resFile))
+        assert type(anns) == list, "results is not an array of objects"
+        annsQuesIds = [ann["question_id"] for ann in anns]
+        assert set(annsQuesIds) == set(
+            self.getQuesIds()
+        ), "Results do not correspond to current VQA set. Either the results do not have predictions for all question ids in annotation file or there is atleast one question id that does not belong to the question ids in the annotation file."
+        for ann in anns:
+            quesId = ann["question_id"]
+            if res.dataset["task_type"] == "Multiple Choice":
+                assert (
+                    ann["answer"] in self.qqa[quesId]["multiple_choices"]
+                ), "predicted answer is not one of the multiple choices"
+            qaAnn = self.qa[quesId]
+            ann["image_id"] = qaAnn["image_id"]
+            ann["question_type"] = qaAnn["question_type"]
+            ann["answer_type"] = qaAnn["answer_type"]
+        print(
+            "DONE (t=%0.2fs)" % ((datetime.datetime.utcnow() - time_t).total_seconds())
+        )
+
+        res.dataset["annotations"] = anns
+        res.createIndex()
+        return res

LAVIS-main/lavis/common/vqa_tools/vqa_eval.py

@@ -0,0 +1,324 @@
+"""
+ Copyright (c) 2022, salesforce.com, inc.
+ All rights reserved.
+ SPDX-License-Identifier: BSD-3-Clause
+ For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+"""
+
+# coding=utf-8
+
+__author__ = "aagrawal"
+
+# This code is based on the code written by Tsung-Yi Lin for MSCOCO Python API available at the following link:
+# (https://github.com/tylin/coco-caption/blob/master/pycocoevalcap/eval.py).
+import sys
+import re
+
+
+class VQAEval:
+    def __init__(self, vqa=None, vqaRes=None, n=2):
+        self.n = n
+        self.accuracy = {}
+        self.evalQA = {}
+        self.evalQuesType = {}
+        self.evalAnsType = {}
+        self.vqa = vqa
+        self.vqaRes = vqaRes
+        if vqa is not None:
+            self.params = {"question_id": vqa.getQuesIds()}
+        self.contractions = {
+            "aint": "ain't",
+            "arent": "aren't",
+            "cant": "can't",
+            "couldve": "could've",
+            "couldnt": "couldn't",
+            "couldn'tve": "couldn't've",
+            "couldnt've": "couldn't've",
+            "didnt": "didn't",
+            "doesnt": "doesn't",
+            "dont": "don't",
+            "hadnt": "hadn't",
+            "hadnt've": "hadn't've",
+            "hadn'tve": "hadn't've",
+            "hasnt": "hasn't",
+            "havent": "haven't",
+            "hed": "he'd",
+            "hed've": "he'd've",
+            "he'dve": "he'd've",
+            "hes": "he's",
+            "howd": "how'd",
+            "howll": "how'll",
+            "hows": "how's",
+            "Id've": "I'd've",
+            "I'dve": "I'd've",
+            "Im": "I'm",
+            "Ive": "I've",
+            "isnt": "isn't",
+            "itd": "it'd",
+            "itd've": "it'd've",
+            "it'dve": "it'd've",
+            "itll": "it'll",
+            "let's": "let's",
+            "maam": "ma'am",
+            "mightnt": "mightn't",
+            "mightnt've": "mightn't've",
+            "mightn'tve": "mightn't've",
+            "mightve": "might've",
+            "mustnt": "mustn't",
+            "mustve": "must've",
+            "neednt": "needn't",
+            "notve": "not've",
+            "oclock": "o'clock",
+            "oughtnt": "oughtn't",
+            "ow's'at": "'ow's'at",
+            "'ows'at": "'ow's'at",
+            "'ow'sat": "'ow's'at",
+            "shant": "shan't",
+            "shed've": "she'd've",
+            "she'dve": "she'd've",
+            "she's": "she's",
+            "shouldve": "should've",
+            "shouldnt": "shouldn't",
+            "shouldnt've": "shouldn't've",
+            "shouldn'tve": "shouldn't've",
+            "somebody'd": "somebodyd",
+            "somebodyd've": "somebody'd've",
+            "somebody'dve": "somebody'd've",
+            "somebodyll": "somebody'll",
+            "somebodys": "somebody's",
+            "someoned": "someone'd",
+            "someoned've": "someone'd've",
+            "someone'dve": "someone'd've",
+            "someonell": "someone'll",
+            "someones": "someone's",
+            "somethingd": "something'd",
+            "somethingd've": "something'd've",
+            "something'dve": "something'd've",
+            "somethingll": "something'll",
+            "thats": "that's",
+            "thered": "there'd",
+            "thered've": "there'd've",
+            "there'dve": "there'd've",
+            "therere": "there're",
+            "theres": "there's",
+            "theyd": "they'd",
+            "theyd've": "they'd've",
+            "they'dve": "they'd've",
+            "theyll": "they'll",
+            "theyre": "they're",
+            "theyve": "they've",
+            "twas": "'twas",
+            "wasnt": "wasn't",
+            "wed've": "we'd've",
+            "we'dve": "we'd've",
+            "weve": "we've",
+            "werent": "weren't",
+            "whatll": "what'll",
+            "whatre": "what're",
+            "whats": "what's",
+            "whatve": "what've",
+            "whens": "when's",
+            "whered": "where'd",
+            "wheres": "where's",
+            "whereve": "where've",
+            "whod": "who'd",
+            "whod've": "who'd've",
+            "who'dve": "who'd've",
+            "wholl": "who'll",
+            "whos": "who's",
+            "whove": "who've",
+            "whyll": "why'll",
+            "whyre": "why're",
+            "whys": "why's",
+            "wont": "won't",
+            "wouldve": "would've",
+            "wouldnt": "wouldn't",
+            "wouldnt've": "wouldn't've",
+            "wouldn'tve": "wouldn't've",
+            "yall": "y'all",
+            "yall'll": "y'all'll",
+            "y'allll": "y'all'll",
+            "yall'd've": "y'all'd've",
+            "y'alld've": "y'all'd've",
+            "y'all'dve": "y'all'd've",
+            "youd": "you'd",
+            "youd've": "you'd've",
+            "you'dve": "you'd've",
+            "youll": "you'll",
+            "youre": "you're",
+            "youve": "you've",
+        }
+        self.manualMap = {
+            "none": "0",
+            "zero": "0",
+            "one": "1",
+            "two": "2",
+            "three": "3",
+            "four": "4",
+            "five": "5",
+            "six": "6",
+            "seven": "7",
+            "eight": "8",
+            "nine": "9",
+            "ten": "10",
+        }
+        self.articles = ["a", "an", "the"]
+
+        self.periodStrip = re.compile("(?!<=\d)(\.)(?!\d)")
+        self.commaStrip = re.compile("(\d)(,)(\d)")
+        self.punct = [
+            ";",
+            r"/",
+            "[",
+            "]",
+            '"',
+            "{",
+            "}",
+            "(",
+            ")",
+            "=",
+            "+",
+            "\\",
+            "_",
+            "-",
+            ">",
+            "<",
+            "@",
+            "`",
+            ",",
+            "?",
+            "!",
+        ]
+
+    def evaluate(self, quesIds=None):
+        if quesIds == None:
+            quesIds = [quesId for quesId in self.params["question_id"]]
+        gts = {}
+        res = {}
+        for quesId in quesIds:
+            gts[quesId] = self.vqa.qa[quesId]
+            res[quesId] = self.vqaRes.qa[quesId]
+
+        # =================================================
+        # Compute accuracy
+        # =================================================
+        accQA = []
+        accQuesType = {}
+        accAnsType = {}
+        print("computing accuracy")
+        step = 0
+        for quesId in quesIds:
+            resAns = res[quesId]["answer"]
+            resAns = resAns.replace("\n", " ")
+            resAns = resAns.replace("\t", " ")
+            resAns = resAns.strip()
+            resAns = self.processPunctuation(resAns)
+            resAns = self.processDigitArticle(resAns)
+            gtAcc = []
+            gtAnswers = [ans["answer"] for ans in gts[quesId]["answers"]]
+            if len(set(gtAnswers)) > 1:
+                for ansDic in gts[quesId]["answers"]:
+                    ansDic["answer"] = self.processPunctuation(ansDic["answer"])
+            for gtAnsDatum in gts[quesId]["answers"]:
+                otherGTAns = [
+                    item for item in gts[quesId]["answers"] if item != gtAnsDatum
+                ]
+                matchingAns = [item for item in otherGTAns if item["answer"] == resAns]
+                acc = min(1, float(len(matchingAns)) / 3)
+                gtAcc.append(acc)
+            quesType = gts[quesId]["question_type"]
+            ansType = gts[quesId]["answer_type"]
+            avgGTAcc = float(sum(gtAcc)) / len(gtAcc)
+            accQA.append(avgGTAcc)
+            if quesType not in accQuesType:
+                accQuesType[quesType] = []
+            accQuesType[quesType].append(avgGTAcc)
+            if ansType not in accAnsType:
+                accAnsType[ansType] = []
+            accAnsType[ansType].append(avgGTAcc)
+            self.setEvalQA(quesId, avgGTAcc)
+            self.setEvalQuesType(quesId, quesType, avgGTAcc)
+            self.setEvalAnsType(quesId, ansType, avgGTAcc)
+            if step % 100 == 0:
+                self.updateProgress(step / float(len(quesIds)))
+            step = step + 1
+
+        self.setAccuracy(accQA, accQuesType, accAnsType)
+        print("Done computing accuracy")
+
+    def processPunctuation(self, inText):
+        outText = inText
+        for p in self.punct:
+            if (p + " " in inText or " " + p in inText) or (
+                re.search(self.commaStrip, inText) != None
+            ):
+                outText = outText.replace(p, "")
+            else:
+                outText = outText.replace(p, " ")
+        outText = self.periodStrip.sub("", outText, re.UNICODE)
+        return outText
+
+    def processDigitArticle(self, inText):
+        outText = []
+        tempText = inText.lower().split()
+        for word in tempText:
+            word = self.manualMap.setdefault(word, word)
+            if word not in self.articles:
+                outText.append(word)
+            else:
+                pass
+        for wordId, word in enumerate(outText):
+            if word in self.contractions:
+                outText[wordId] = self.contractions[word]
+        outText = " ".join(outText)
+        return outText
+
+    def setAccuracy(self, accQA, accQuesType, accAnsType):
+        self.accuracy["overall"] = round(100 * float(sum(accQA)) / len(accQA), self.n)
+        self.accuracy["perQuestionType"] = {
+            quesType: round(
+                100 * float(sum(accQuesType[quesType])) / len(accQuesType[quesType]),
+                self.n,
+            )
+            for quesType in accQuesType
+        }
+        self.accuracy["perAnswerType"] = {
+            ansType: round(
+                100 * float(sum(accAnsType[ansType])) / len(accAnsType[ansType]), self.n
+            )
+            for ansType in accAnsType
+        }
+
+    def setEvalQA(self, quesId, acc):
+        self.evalQA[quesId] = round(100 * acc, self.n)
+
+    def setEvalQuesType(self, quesId, quesType, acc):
+        if quesType not in self.evalQuesType:
+            self.evalQuesType[quesType] = {}
+        self.evalQuesType[quesType][quesId] = round(100 * acc, self.n)
+
+    def setEvalAnsType(self, quesId, ansType, acc):
+        if ansType not in self.evalAnsType:
+            self.evalAnsType[ansType] = {}
+        self.evalAnsType[ansType][quesId] = round(100 * acc, self.n)
+
+    def updateProgress(self, progress):
+        barLength = 20
+        status = ""
+        if isinstance(progress, int):
+            progress = float(progress)
+        if not isinstance(progress, float):
+            progress = 0
+            status = "error: progress var must be float\r\n"
+        if progress < 0:
+            progress = 0
+            status = "Halt...\r\n"
+        if progress >= 1:
+            progress = 1
+            status = "Done...\r\n"
+        block = int(round(barLength * progress))
+        text = "\rFinshed Percent: [{0}] {1}% {2}".format(
+            "#" * block + "-" * (barLength - block), int(progress * 100), status
+        )
+        sys.stdout.write(text)
+        sys.stdout.flush()

LAVIS-main/lavis/configs/datasets/aokvqa/defaults.yaml

@@ -0,0 +1,35 @@
+ # Copyright (c) 2022, salesforce.com, inc.
+ # All rights reserved.
+ # SPDX-License-Identifier: BSD-3-Clause
+ # For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+
+datasets:
+  aok_vqa:
+    # data_dir: ${env.data_dir}/datasets
+    data_type: images # [images|videos|features]
+
+    build_info:
+      # Be careful not to append minus sign (-) before split to avoid itemizing
+      annotations:
+        train:
+          url:
+              - https://storage.googleapis.com/sfr-vision-language-research/LAVIS/datasets/aokvqa/aokvqa_v1p0_train.json
+          storage:
+              - aokvqa/annotations/aokvqa_v1p0_train.json
+        val:
+          url:
+              - https://storage.googleapis.com/sfr-vision-language-research/LAVIS/datasets/aokvqa/aokvqa_v1p0_val.json
+              - https://storage.googleapis.com/sfr-vision-language-research/LAVIS/datasets/aokvqa/specialized_vocab_train.json
+          storage:
+              - aokvqa/annotations/aokvqa_v1p0_val.json
+              - aokvqa/annotations/specialized_vocab_train_lavis.json
+              # - aokvqa/annotations/large_vocab_train_lavis.json
+        test:
+          url:
+              - https://storage.googleapis.com/sfr-vision-language-research/LAVIS/datasets/aokvqa/aokvqa_v1p0_test.json
+              - https://storage.googleapis.com/sfr-vision-language-research/LAVIS/datasets/aokvqa/specialized_vocab_train.json
+          storage:
+              - aokvqa/annotations/aokvqa_v1p0_test.json
+              - aokvqa/annotations/specialized_vocab_train_lavis.json
+      images:
+          storage: coco/images/

LAVIS-main/lavis/configs/datasets/aokvqa/defaults_instruct.yaml

@@ -0,0 +1,52 @@
+ # Copyright (c) 2023, salesforce.com, inc.
+ # All rights reserved.
+ # SPDX-License-Identifier: BSD-3-Clause
+ # For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+
+datasets:
+  aok_vqa_instruct:
+    # data_dir: ${env.data_dir}/datasets
+    data_type: images # [images|videos|features]
+
+    vis_processor:
+        train:
+          name: "clip_image_train"
+          image_size: 224
+        eval:
+          name: "clip_image_eval"
+          image_size: 224
+
+    text_processor:
+        train:
+          name: blip_instruction
+          modality: image
+          task: qa
+        eval:
+          name: blip_question
+
+    build_info:
+      # Be careful not to append minus sign (-) before split to avoid itemizing
+      annotations:
+        train:
+          url:
+              - https://storage.googleapis.com/sfr-vision-language-research/LAVIS/datasets/aokvqa/aokvqa_v1p0_train.json
+          storage:
+              - aokvqa/annotations/aokvqa_v1p0_train.json
+        # val:
+        #   url:
+        #       - https://storage.googleapis.com/sfr-vision-language-research/LAVIS/datasets/aokvqa/aokvqa_v1p0_val.json
+        #       - https://storage.googleapis.com/sfr-vision-language-research/LAVIS/datasets/aokvqa/specialized_vocab_train.json
+        #   storage:
+        #       - aokvqa/annotations/aokvqa_v1p0_val.json
+        #       - aokvqa/annotations/specialized_vocab_train_lavis.json
+        #       # - aokvqa/annotations/large_vocab_train_lavis.json
+        # test:
+        #   url:
+        #       - https://storage.googleapis.com/sfr-vision-language-research/LAVIS/datasets/aokvqa/aokvqa_v1p0_test.json
+        #       - https://storage.googleapis.com/sfr-vision-language-research/LAVIS/datasets/aokvqa/specialized_vocab_train.json
+        #   storage:
+        #       - aokvqa/annotations/aokvqa_v1p0_test.json
+        #       - aokvqa/annotations/specialized_vocab_train_lavis.json
+      images:
+          # storage: /coco/images
+          storage: /export/share/datasets/vision/coco/images

LAVIS-main/lavis/configs/datasets/audiocaps/defaults_mm_cap.yaml

@@ -0,0 +1,49 @@
+ # Copyright (c) 2023, salesforce.com, inc.
+ # All rights reserved.
+ # SPDX-License-Identifier: BSD-3-Clause
+ # For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+
+datasets:
+  audiocaps_mm_caption: # name of the dataset builder
+    audio_processor:
+        train:
+          name: beats_audio
+          sampling_rate: 16000
+        eval:
+          name: beats_audio
+          sampling_rate: 16000
+          
+    text_processor:
+        train:
+          name: "blip_instruction"
+          modality: audio
+          task: caption
+        eval:
+          name: "blip_caption"
+
+    data_type: [audio]
+
+    build_info:
+      kwargs:
+            missing_ids: [2sh7ZkazyO8, 966jA2-z0mQ, 52RlolYyjAE, HVAc9hm4jjk, 8lPjqvYWNyM, eXgPnnE3TuQ]
+      annotations:
+        train:
+          url: 
+            - https://raw.githubusercontent.com/cdjkim/audiocaps/master/dataset/train.csv
+          storage: 
+            - audiocaps/annotations/train.csv
+          
+        val:
+          url: 
+            - https://raw.githubusercontent.com/cdjkim/audiocaps/master/dataset/val.csv
+          storage: 
+            - audiocaps/annotations/val.csv
+
+        test:
+          url: 
+            - https://raw.githubusercontent.com/cdjkim/audiocaps/master/dataset/test.csv
+          storage: 
+            - audiocaps/annotations/test.csv
+            
+      audio:
+        storage: /export/einstein-vision/audio_datasets/audiocaps/AUDIOCAPS_32000Hz/audio

LAVIS-main/lavis/configs/datasets/audiocaps/defaults_mm_cap_instruct.yaml

@@ -0,0 +1,52 @@
+ # Copyright (c) 2023, salesforce.com, inc.
+ # All rights reserved.
+ # SPDX-License-Identifier: BSD-3-Clause
+ # For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+
+datasets:
+  audiocaps_mm_caption_instruct: # name of the dataset builder
+    audio_processor:
+        train:
+          name: beats_audio
+          sampling_rate: 16000
+        eval:
+          name: beats_audio
+          sampling_rate: 16000
+          
+    text_processor:
+        train:
+          name: "blip_instruction"
+          modality: audio
+          task: caption
+        eval:
+          name: "blip_caption"
+
+    data_type: [audio]
+
+    missing_ids: [2sh7ZkazyO8, 966jA2-z0mQ, 52RlolYyjAE, HVAc9hm4jjk, 8lPjqvYWNyM, eXgPnnE3TuQ]
+
+    build_info:
+      kwargs:
+        cached: False
+        cached_dir: /export/einstein-vision/audio_datasets/audiocaps/beats_features
+      annotations:
+        train:
+          url: 
+            - https://raw.githubusercontent.com/cdjkim/audiocaps/master/dataset/train.csv
+          storage: 
+            - audiocaps/annotations/train.csv
+          
+        # val:
+        #   url: 
+        #     - https://raw.githubusercontent.com/cdjkim/audiocaps/master/dataset/val.csv
+        #   storage: 
+        #     - audiocaps/annotation/val.csv
+
+        # test:
+        #   url: 
+        #     - https://raw.githubusercontent.com/cdjkim/audiocaps/master/dataset/test.csv
+        #   storage: 
+        #     - /export/einstein-vision/audio_datasets/audiocaps/dataset/test.csv
+            
+      audio:
+        storage: /export/einstein-vision/audio_datasets/audiocaps/AUDIOCAPS_32000Hz/audio

LAVIS-main/lavis/configs/datasets/audiocaps/defaults_mm_qa.yaml

@@ -0,0 +1,51 @@
+ # Copyright (c) 2023, salesforce.com, inc.
+ # All rights reserved.
+ # SPDX-License-Identifier: BSD-3-Clause
+ # For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+
+datasets:
+  audiocaps_mm_qa: # name of the dataset builder
+    audio_processor:
+        train:
+          name: beats_audio
+          sampling_rate: 16000
+        eval:
+          name: beats_audio
+          sampling_rate: 16000
+          is_eval: True
+          
+    text_processor:
+        train:
+          name: "blip_instruction"
+          modality: audio
+          task: qa
+        eval:
+          name: "blip_question"
+
+    data_type: [audio] 
+
+    build_info:
+      kwargs:
+        cached: False
+        # add_binary: True
+        cached_dir: /export/einstein-vision/audio_datasets/audiocaps/beats_features
+        missing_ids: [2sh7ZkazyO8, 966jA2-z0mQ, 52RlolYyjAE, HVAc9hm4jjk, 8lPjqvYWNyM, eXgPnnE3TuQ]
+      annotations:
+        train:
+          url: 
+            - https://storage.googleapis.com/sfr-xinstructblip-data-research/data/audiocaps/audio_qa_final_train.csv
+            # - /export/home/LAVIS-xgen_mm/projects/xinstructblip/data_aug/audio_qa_data/audio_qa_final_train.csv
+          storage: 
+            - audiocaps_qa/annotations/train.csv
+            # - /export/home/LAVIS-xgen_mm/projects/xinstructblip/data_aug/audio_qa_data/audio_qa_final_train.csv
+          
+        # val:
+        #   url: 
+        #     # - https://storage.googleapis.com/sfr-xinstructblip-data-research/data/audiocaps/audio_qa_final_val.csv
+        #     - /export/home/LAVIS-xgen_mm/projects/xinstructblip/data_aug/audio_qa_data/audio_qa_final_val.csv
+        #   storage: 
+        #     # - audiocaps_qa/annotations/val.csv
+        #     - /export/home/LAVIS-xgen_mm/projects/xinstructblip/data_aug/audio_qa_data/audio_qa_final_val.csv
+
+      audio:
+        storage: /export/einstein-vision/audio_datasets/audiocaps/AUDIOCAPS_32000Hz/audio

LAVIS-main/lavis/configs/datasets/audioset/defaults_mm_cap.yaml

@@ -0,0 +1,47 @@
+ # Copyright (c) 2023, salesforce.com, inc.
+ # All rights reserved.
+ # SPDX-License-Identifier: BSD-3-Clause
+ # For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+
+datasets:
+  audioset_mm_caption: # 14141
+    audio_processor:
+        train:
+          name: beats_audio
+          sampling_rate: 16000
+        eval:
+          name: beats_audio
+          sampling_rate: 16000
+          is_eval: False
+          
+    text_processor:
+        train:
+          name: blip_instruction
+          modality: audio
+          task: classification
+        eval:
+          name: blip_caption
+
+    data_type: [audio] 
+
+    build_info:
+      annotations:
+        train:
+          url: 
+            - https://storage.googleapis.com/sfr-xinstructblip-data-research/data//audioset/balanced_train_clean.csv
+            # - /export/home/LAVIS-xgen_mm/lavis/configs/datasets/audioset/balanced_train_clean.csv
+            - http://storage.googleapis.com/us_audioset/youtube_corpus/v1/csv/class_labels_indices.csv
+          storage:
+            - audioset/balanced_train_clean.csv
+            # - /export/home/LAVIS-xgen_mm/lavis/configs/datasets/audioset/balanced_train_clean.csv
+            - audioset/annotations/class_labels_indices.csv
+             
+        # val:
+        #   url: 
+        #     - http://storage.googleapis.com/us_audioset/youtube_corpus/v1/csv/eval_segments.csv
+        #     - http://storage.googleapis.com/us_audioset/youtube_corpus/v1/csv/class_labels_indices.csv
+        #   storage: 
+        #     - audioset/annotations/eval_segments.csv
+        #     - audioset/annotations/class_labels_indices.csv
+      audio:
+        storage: /export/einstein-vision/audio_datasets/AudioSet/all_audio

LAVIS-main/lavis/configs/datasets/audioset/defaults_mm_cap_instruct.yaml

@@ -0,0 +1,48 @@
+ # Copyright (c) 2023, salesforce.com, inc.
+ # All rights reserved.
+ # SPDX-License-Identifier: BSD-3-Clause
+ # For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+
+datasets:
+  audioset_mm_caption_instruct: # 14141
+    audio_processor:
+        train:
+          name: beats_audio
+          sampling_rate: 16000
+        eval:
+          name: beats_audio
+          sampling_rate: 16000
+          is_eval: False
+          
+    text_processor:
+        train:
+          name: blip_instruction
+          modality: audio
+          task: classification
+        eval:
+          name: blip_caption
+
+    data_type: [audio] 
+
+    build_info:
+      annotations:
+        train:
+          url: 
+            # - https://storage.googleapis.com/sfr-xinstructblip-data-research/data//audioset/balanced_train_clean.csv
+            - /export/home/LAVIS-xgen_mm/lavis/configs/datasets/audioset/balanced_train_clean.csv
+            - http://storage.googleapis.com/us_audioset/youtube_corpus/v1/csv/class_labels_indices.csv
+          storage:
+            - audioset/annotations/balanced_train_clean.csv
+            # - /export/home/LAVIS-xgen_mm/lavis/configs/datasets/audioset/balanced_train_clean.csv
+            - audioset/annotations/class_labels_indices.csv
+             
+        # val:
+        #   url: 
+        #     - http://storage.googleapis.com/us_audioset/youtube_corpus/v1/csv/eval_segments.csv
+        #     - http://storage.googleapis.com/us_audioset/youtube_corpus/v1/csv/class_labels_indices.csv
+        #   storage: 
+        #     - audioset/annotations/eval_segments.csv
+        #     - audioset/annotations/class_labels_indices.csv
+        
+      audio:
+        storage: /export/einstein-vision/audio_datasets/AudioSet/all_audio

LAVIS-main/lavis/configs/datasets/avsd/defaults_dial.yaml

@@ -0,0 +1,24 @@
+ # Copyright (c) 2022, salesforce.com, inc.
+ # All rights reserved.
+ # SPDX-License-Identifier: BSD-3-Clause
+ # For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+
+datasets:
+  avsd_dialogue: # name of the dataset builder
+    dataset_card: dataset_card/avsd_dialogue.md 
+    data_type: features #extracted features of videos (I3D, VGGish) # [images|videos|features]
+
+    build_info:
+      # Be careful not to append minus sign (-) before split to avoid itemizing
+      annotations:
+        train:
+          url: https://storage.googleapis.com/sfr-vision-language-research/datasets/avsd_dstc7_train.json
+          storage: avsd/annotations/train.json 
+        val:
+          url: https://storage.googleapis.com/sfr-vision-language-research/datasets/avsd_dstc7_val.json
+          storage: avsd/annotations/val.json 
+        test:
+          url: https://storage.googleapis.com/sfr-vision-language-research/datasets/avsd_dstc7_test.json
+          storage: avsd/annotations/test.json 
+      features:
+        storage: avsd/features/ 

LAVIS-main/lavis/configs/datasets/avsd/defaults_mm_dial_instruct.yaml

@@ -0,0 +1,65 @@
+ # Copyright (c) 2023, salesforce.com, inc.
+ # All rights reserved.
+ # SPDX-License-Identifier: BSD-3-Clause
+ # For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+
+datasets:
+  avsd_mm_dialogue_instruct: # name of the dataset builder
+    data_type: [video, audio]
+
+    audio_processor:
+      train:
+        name: beats_audio
+        sampling_rate: 16000
+      eval:
+        name: beats_audio
+        sampling_rate: 16000
+
+    video_processor:
+        train:
+          name: alpro_video_train
+          n_frms: 4
+          image_size: 224
+          min_scale: 0.9
+          max_scale: 1.0
+          full_video: True
+        eval:
+          name: alpro_video_eval
+          n_frms: 4
+          image_size: 224
+          min_scale: 0.9
+          max_scale: 1.0
+          full_video: True
+        
+    text_processor:
+      train:
+        name: "blip_caption"
+      eval:
+        name: "blip_caption"
+
+    build_info:
+      # Be careful not to append minus sign (-) before split to avoid itemizing
+      annotations:
+        train:
+          url: 
+            - https://storage.googleapis.com/sfr-vision-language-research/datasets/avsd_dstc7_train.json
+          storage: 
+            - avsd/annotations/train.json 
+        val:
+          url: 
+            - https://storage.googleapis.com/sfr-vision-language-research/datasets/avsd_dstc7_val.json
+          storage: 
+            - avsd/annotations/val.json 
+        test:
+          url: 
+            - https://storage.googleapis.com/sfr-vision-language-research/datasets/avsd_dstc7_test.json
+          storage: 
+            - avsd/annotations/test.json 
+      templates: null
+
+      audio:
+        storage: /export/video-language-dataset/data/charade/videos
+
+      video:
+        storage: /export/video-language-dataset/data/charade/videos
+

LAVIS-main/lavis/configs/datasets/blip_diffusion_datasets/defaults.yaml

@@ -0,0 +1,14 @@
+ # Copyright (c) 2022, salesforce.com, inc.
+ # All rights reserved.
+ # SPDX-License-Identifier: BSD-3-Clause
+ # For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+
+datasets:
+  blip_diffusion_finetune: # name of the dataset builder
+    # data_dir: ${env.data_dir}/datasets
+    data_type: images # [images|videos|features]
+
+    build_info:
+      # Be careful not to append minus sign (-) before split to avoid itemizing
+      images:
+        storage: ""

LAVIS-main/lavis/configs/datasets/capfilt14m/defaults_cap.yaml

@@ -0,0 +1,30 @@
+ # Copyright (c) 2023, salesforce.com, inc.
+ # All rights reserved.
+ # SPDX-License-Identifier: BSD-3-Clause
+ # For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+
+datasets:
+  capfilt14m: # 13873136
+    # data_dir: ${env.data_dir}/datasets
+    data_type: images # [images|videos|features]
+
+    vis_processor:
+        train:
+          name: "clip_image_train"
+          image_size: 224
+    text_processor:
+        train:
+          name: blip_caption
+
+    build_info:
+      # Be careful not to append minus sign (-) before split to avoid itemizing
+      annotations:
+        train:
+          url: 
+            - https://storage.googleapis.com/sfr-xinstructblip-data-research/data/capfilt14m/annotation.json
+            # - /export/share/datasets/vision_language/capfilt_14m_new/annotation.json
+          storage: 
+            - capfilt14m/annotations/annotation.json
+            # - /export/share/datasets/vision_language/capfilt_14m_new/annotation.json
+      images:
+        storage: /export/share/datasets/vision/coco/images

LAVIS-main/lavis/configs/datasets/capfilt14m/defaults_cap_instruct.yaml

@@ -0,0 +1,34 @@
+ # Copyright (c) 2023, salesforce.com, inc.
+ # All rights reserved.
+ # SPDX-License-Identifier: BSD-3-Clause
+ # For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+
+datasets:
+  capfilt14m_instruct: # 13873136
+    # data_dir: ${env.data_dir}/datasets
+    data_type: images # [images|videos|features]
+
+    vis_processor:
+        train:
+          name: "clip_image_train"
+          image_size: 224
+
+    text_processor:
+        train:
+          name: blip_instruction
+          modality: image
+          task: caption
+
+    build_info:
+      # Be careful not to append minus sign (-) before split to avoid itemizing
+      annotations:
+        train:
+          url: 
+            - https://storage.googleapis.com/sfr-xinstructblip-data-research/data/capfilt14m/annotation.json
+            # - /export/share/datasets/vision_language/capfilt_14m_new/annotation.json
+          storage: 
+            - capfilt14m/annotations/annotation.json
+            # - /export/share/datasets/vision_language/capfilt_14m_new/annotation.json
+
+      images:
+        storage: /export/share/datasets/vision/coco/images

LAVIS-main/lavis/configs/datasets/charade/defaults_cap.yaml

@@ -0,0 +1,52 @@
+ # Copyright (c) 2023, salesforce.com, inc.
+ # All rights reserved.
+ # SPDX-License-Identifier: BSD-3-Clause
+ # For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+
+datasets:
+  charade_caption: # name of the dataset builder
+    # data_dir: ${env.data_dir}/datasets
+    data_type: videos # [images|videos|features]
+
+    vis_processor:
+        train:
+          name: alpro_video_train
+          n_frms: 4
+          image_size: 224
+          min_scale: 0.9
+          max_scale: 1.0
+          full_video: False
+        eval:
+          name: alpro_video_eval
+          n_frms: 4
+          image_size: 224
+          min_scale: 0.9
+          max_scale: 1.0
+          full_video: False
+    
+    text_processor:
+        train:
+          name: blip_caption
+        eval:
+          name: blip_caption
+      
+
+    build_info:
+      # Be careful not to append minus sign (-) before split to avoid itemizing
+      annotations:
+        train:
+          url: 
+            - https://storage.googleapis.com/sfr-xinstructblip-data-research/data/charade/train_lavis.json
+            # - /export/video-language-dataset/data/charade/train_lavis.json
+          storage: 
+            - charade/annotations/train.json
+            # - /export/video-language-dataset/data/charade/train_lavis.json
+        val:
+          url: 
+            - https://storage.googleapis.com/sfr-xinstructblip-data-research/data/charade/val_lavis.json
+            # - /export/video-language-dataset/data/charade/val_lavis.json
+          storage: 
+            - charade/annotations/val.json
+            # - /export/video-language-dataset/data/charade/val_lavis.json
+      videos:
+        storage: /export/video-language-dataset/data/charade/videos

LAVIS-main/lavis/configs/datasets/charade/defaults_cap_instruct.yaml

@@ -0,0 +1,54 @@
+ # Copyright (c) 2023, salesforce.com, inc.
+ # All rights reserved.
+ # SPDX-License-Identifier: BSD-3-Clause
+ # For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+
+datasets:
+  charade_caption_instruct: # name of the dataset builder
+    # data_dir: ${env.data_dir}/datasets
+    data_type: videos # [images|videos|features]
+
+    vis_processor:
+        train:
+          name: alpro_video_train
+          n_frms: 4
+          image_size: 224
+          min_scale: 0.9
+          max_scale: 1.0
+          full_video: False
+        eval:
+          name: alpro_video_eval
+          n_frms: 4
+          image_size: 224
+          min_scale: 0.9
+          max_scale: 1.0
+          full_video: False
+    
+
+    text_processor:
+        train:
+          name: blip_instruction
+          modality: video
+          task: caption
+        eval:
+          name: blip_caption
+
+    build_info:
+      # Be careful not to append minus sign (-) before split to avoid itemizing
+      annotations:
+        train:
+          url: 
+            - https://storage.googleapis.com/sfr-xinstructblip-data-research/data/charade/train_lavis.json
+            # - /export/video-language-dataset/data/charade/train_lavis.json
+          storage: 
+            - charade/annotations/train.json
+            # - /export/video-language-dataset/data/charade/train_lavis.json
+        val:
+          url: 
+            - https://storage.googleapis.com/sfr-xinstructblip-data-research/data/charade/val_lavis.json
+            # - /export/video-language-dataset/data/charade/val_lavis.json
+          storage: 
+            - charade/annotations/val.json
+            # - /export/video-language-dataset/data/charade/val_lavis.json
+      videos:
+        storage: /export/video-language-dataset/data/charade/videos

LAVIS-main/lavis/configs/datasets/clotho/defaults_mm_cap.yaml

@@ -0,0 +1,41 @@
+ # Copyright (c) 2023, salesforce.com, inc.
+ # All rights reserved.
+ # SPDX-License-Identifier: BSD-3-Clause
+ # For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+
+datasets:
+  clothov2: # name of the dataset builder
+    audio_processor:
+        train:
+          name: beats_audio
+        eval:
+          name: beats_audio
+          
+    text_processor:
+        train:
+          name: "blip_caption"
+        eval:
+          name: "blip_caption"
+    
+
+    data_type: [audio] 
+
+    build_info:
+      kwargs:
+        clotho_root: /export/einstein-vision/audio_datasets/clothov2/
+        split: eval
+
+      annotations:
+        train:
+          url: 
+            - https://zenodo.org/record/4783391/files/clotho_captions_development.csv
+          storage: 
+            - clothov2/annotations/clotho_captions_development.csv
+        val:
+          url: 
+            - https://zenodo.org/record/4783391/files/clotho_captions_evaluation.csv
+          storage: 
+            - clothov2/annotations/clotho_captions_evaluation.csv
+      audio:
+        storage: /export/einstein-vision/audio_datasets/clothov2/CLOTHO_v2.1/clotho_audio_files/
+  

LAVIS-main/lavis/configs/datasets/clotho/defaults_mm_cap_instruct.yaml

@@ -0,0 +1,42 @@
+ # Copyright (c) 2023, salesforce.com, inc.
+ # All rights reserved.
+ # SPDX-License-Identifier: BSD-3-Clause
+ # For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+
+datasets:
+  clothov2_instruct: # name of the dataset builder
+    audio_processor:
+        train:
+          name: beats_audio
+        eval:
+          name: beats_audio
+          
+    text_processor:
+        train:
+          name: "blip_instruction"
+          modality: audio
+          task: caption
+        eval:
+          name: "blip_caption"
+
+    data_type: [audio] 
+
+    build_info:
+      kwargs:
+        clotho_root: /export/einstein-vision/audio_datasets/clothov2/
+        split: eval
+        
+      annotations:
+        train:
+          url: 
+            - https://zenodo.org/record/4783391/files/clotho_captions_development.csv
+          storage: 
+            - clothov2/annotations/clotho_captions_development.csv
+        val:
+          url: 
+            - https://zenodo.org/record/4783391/files/clotho_captions_evaluation.csv
+          storage: 
+            - clothov2/annotations/clotho_captions_evaluation.csv
+      audio:
+        storage: /export/einstein-vision/audio_datasets/clothov2/CLOTHO_v2.1/clotho_audio_files/
+  

LAVIS-main/lavis/configs/datasets/clotho/defaults_mm_qa.yaml

@@ -0,0 +1,44 @@
+ # Copyright (c) 2023, salesforce.com, inc.
+ # All rights reserved.
+ # SPDX-License-Identifier: BSD-3-Clause
+ # For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+
+datasets:
+  clotho_qa: # name of the dataset builder
+    audio_processor:
+        train:
+          name: beats_audio
+        eval:
+          name: beats_audio
+          
+    text_processor:
+        train:
+          name: "blip_caption"
+        eval:
+          name: "blip_caption"
+    
+
+    data_type: [audio] 
+
+    build_info:
+
+      annotations:
+        train:
+          url: 
+            - https://zenodo.org/records/6473207/files/clotho_aqa_train.csv
+          storage: 
+            - clotho_Qa/annotations/clotho_aqa_train.csv
+        val:
+          url: 
+            - https://zenodo.org/records/6473207/files/clotho_aqa_val.csv
+          storage: 
+            - clotho_qa/annotations/clotho_aqa_val.csv
+        
+        test:
+          url: 
+            - https://zenodo.org/records/6473207/files/clotho_aqa_test.csv
+          storage: 
+            - clotho_qa/annotations/clotho_aqa_test.csv
+      audio:
+        storage:  /export/einstein-vision/audio_datasets/clotho-aqa/audio_files
+  

LAVIS-main/lavis/configs/datasets/coco/defaults_cap.yaml

@@ -0,0 +1,28 @@
+ # Copyright (c) 2022, salesforce.com, inc.
+ # All rights reserved.
+ # SPDX-License-Identifier: BSD-3-Clause
+ # For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+
+datasets:
+  coco_caption: # name of the dataset builder
+    dataset_card: dataset_card/coco_caption.md
+    # data_dir: ${env.data_dir}/datasets
+    data_type: images # [images|videos|features]
+
+    build_info:
+      # Be careful not to append minus sign (-) before split to avoid itemizing
+      annotations:
+        train:
+          url: https://storage.googleapis.com/sfr-vision-language-research/datasets/coco_karpathy_train.json
+          md5: aa31ac474cf6250ebb81d18348a07ed8
+          storage: coco/annotations/coco_karpathy_train.json
+        val:
+          url: https://storage.googleapis.com/sfr-vision-language-research/datasets/coco_karpathy_val.json
+          md5: b273847456ef5580e33713b1f7de52a0
+          storage:  coco/annotations/coco_karpathy_val.json
+        test:
+          url: https://storage.googleapis.com/sfr-vision-language-research/datasets/coco_karpathy_test.json
+          md5: 3ff34b0ef2db02d01c37399f6a2a6cd1
+          storage: coco/annotations/coco_karpathy_test.json
+      images:
+        storage: coco/images/

LAVIS-main/lavis/configs/datasets/coco/defaults_cap_instruct.yaml

@@ -0,0 +1,44 @@
+ # Copyright (c) 2023, salesforce.com, inc.
+ # All rights reserved.
+ # SPDX-License-Identifier: BSD-3-Clause
+ # For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+
+datasets:
+  coco_caption_instruct: # name of the dataset builder
+    dataset_card: dataset_card/coco_caption.md
+    # data_dir: ${env.data_dir}/datasets
+    data_type: images # [images|videos|features]
+
+    vis_processor:
+      train:
+        name: "clip_image_train"
+        image_size: 224
+      eval:
+        name: "clip_image_eval"
+        image_size: 224
+    
+    text_processor:
+        train:
+          name: blip_instruction
+          modality: image
+          task: caption
+        eval:
+          name: blip_caption
+        
+    build_info:
+      # Be careful not to append minus sign (-) before split to avoid itemizing
+      annotations:
+        train:
+          url: https://storage.googleapis.com/sfr-vision-language-research/datasets/coco_karpathy_train.json
+          md5: aa31ac474cf6250ebb81d18348a07ed8
+          storage: coco/annotations/coco_karpathy_train.json
+        # val:
+        #   url: https://storage.googleapis.com/sfr-vision-language-research/datasets/coco_karpathy_val.json
+        #   md5: b273847456ef5580e33713b1f7de52a0
+        #   storage:  coco/annotations/coco_karpathy_val.json
+        # test:
+        #   url: https://storage.googleapis.com/sfr-vision-language-research/datasets/coco_karpathy_test.json
+        #   md5: 3ff34b0ef2db02d01c37399f6a2a6cd1
+        #   storage: coco/annotations/coco_karpathy_test.json
+      images:
+        storage: /export/share/datasets/vision/coco/images

LAVIS-main/lavis/configs/datasets/coco/defaults_ret.yaml

@@ -0,0 +1,27 @@
+ # Copyright (c) 2022, salesforce.com, inc.
+ # All rights reserved.
+ # SPDX-License-Identifier: BSD-3-Clause
+ # For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+
+datasets:
+  coco_retrieval:
+    # data_dir: ${env.data_dir}/datasets
+    data_type: images # [images|videos|features]
+
+    build_info:
+      # Be careful not to append minus sign (-) before split to avoid itemizing
+      annotations:
+        train:
+          url: https://storage.googleapis.com/sfr-vision-language-research/datasets/coco_karpathy_train.json
+          md5: aa31ac474cf6250ebb81d18348a07ed8
+          storage: coco/annotations/coco_karpathy_train.json
+        val:
+          url: https://storage.googleapis.com/sfr-vision-language-research/datasets/coco_karpathy_val.json
+          md5: b273847456ef5580e33713b1f7de52a0
+          storage:  coco/annotations/coco_karpathy_val.json
+        test:
+          url: https://storage.googleapis.com/sfr-vision-language-research/datasets/coco_karpathy_test.json
+          md5: 3ff34b0ef2db02d01c37399f6a2a6cd1
+          storage: coco/annotations/coco_karpathy_test.json
+      images:
+          storage: coco/images/

LAVIS-main/lavis/configs/datasets/coco/defaults_vqa.yaml

@@ -0,0 +1,41 @@
+ # Copyright (c) 2022, salesforce.com, inc.
+ # All rights reserved.
+ # SPDX-License-Identifier: BSD-3-Clause
+ # For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+
+datasets:
+  coco_vqa:
+    # data_dir: ${env.data_dir}/datasets
+    data_type: images # [images|videos|features]
+
+    build_info:
+      # Be careful not to append minus sign (-) before split to avoid itemizing
+      annotations:
+        train:
+          url:
+              - https://storage.googleapis.com/sfr-vision-language-research/LAVIS/datasets/vqav2/vqa_train.json
+              - https://storage.googleapis.com/sfr-vision-language-research/LAVIS/datasets/vqav2/vqa_val.json
+          storage:
+              - coco/annotations/vqa_train.json
+              - coco/annotations/vqa_val.json
+        val:
+          url:
+              # TODO make this order insensitive
+              - https://storage.googleapis.com/sfr-vision-language-research/LAVIS/datasets/vqav2/vqa_val_eval.json
+              - https://storage.googleapis.com/sfr-vision-language-research/LAVIS/datasets/vqav2/answer_list.json
+              - https://storage.googleapis.com/sfr-vision-language-research/LAVIS/datasets/vqav2/v2_OpenEnded_mscoco_val2014_questions.json
+              - https://storage.googleapis.com/sfr-vision-language-research/LAVIS/datasets/vqav2/v2_mscoco_val2014_annotations.json
+          storage:
+              - coco/annotations/vqa_val_eval.json
+              - coco/annotations/answer_list.json
+              - coco/annotations/v2_OpenEnded_mscoco_val2014_questions.json
+              - coco/annotations/v2_mscoco_val2014_annotations.json
+        test:
+          url:
+              - https://storage.googleapis.com/sfr-vision-language-research/LAVIS/datasets/vqav2/vqa_test.json
+              - https://storage.googleapis.com/sfr-vision-language-research/LAVIS/datasets/vqav2/answer_list.json
+          storage:
+              - coco/annotations/vqa_test.json
+              - coco/annotations/answer_list.json
+      images:
+          storage: coco/images/

LAVIS-main/lavis/configs/datasets/coco/defaults_vqa_instruct.yaml

@@ -0,0 +1,57 @@
+ # Copyright (c) 2023, salesforce.com, inc.
+ # All rights reserved.
+ # SPDX-License-Identifier: BSD-3-Clause
+ # For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+
+datasets:
+  coco_vqa_instruct:
+    # data_dir: ${env.data_dir}/datasets
+    data_type: images # [images|videos|features]
+    
+    vis_processor:
+      train:
+        name: "clip_image_train"
+        image_size: 224
+      eval:
+        name: "clip_image_eval"
+        image_size: 224
+    
+    text_processor:
+      train:
+        name: blip_instruction
+        modality: image
+        task: qa
+      eval:
+        name: blip_caption
+
+    build_info:
+      # Be careful not to append minus sign (-) before split to avoid itemizing
+      annotations:
+        train:
+          url:
+              - https://storage.googleapis.com/sfr-vision-language-research/LAVIS/datasets/vqav2/vqa_train.json
+              - https://storage.googleapis.com/sfr-vision-language-research/LAVIS/datasets/vqav2/vqa_val.json
+          storage:
+              - coco/annotations/vqa_train.json
+              - coco/annotations/vqa_val.json
+        # val:
+        #   url:
+        #       # TODO make this order insensitive
+        #       - https://storage.googleapis.com/sfr-vision-language-research/LAVIS/datasets/vqav2/vqa_val_eval.json
+        #       - https://storage.googleapis.com/sfr-vision-language-research/LAVIS/datasets/vqav2/answer_list.json
+        #       - https://storage.googleapis.com/sfr-vision-language-research/LAVIS/datasets/vqav2/v2_OpenEnded_mscoco_val2014_questions.json
+        #       - https://storage.googleapis.com/sfr-vision-language-research/LAVIS/datasets/vqav2/v2_mscoco_val2014_annotations.json
+        #   storage:
+        #       - coco/annotations/vqa_val_eval.json
+        #       - coco/annotations/answer_list.json
+        #       - coco/annotations/v2_OpenEnded_mscoco_val2014_questions.json
+        #       - coco/annotations/v2_mscoco_val2014_annotations.json
+        # test:
+        #   url:
+        #       - https://storage.googleapis.com/sfr-vision-language-research/LAVIS/datasets/vqav2/vqa_test.json
+        #       - https://storage.googleapis.com/sfr-vision-language-research/LAVIS/datasets/vqav2/answer_list.json
+        #   storage:
+        #       - coco/annotations/vqa_test.json
+        #       - coco/annotations/answer_list.json
+      images:
+          storage: /export/share/datasets/vision/coco/images

LAVIS-main/lavis/configs/datasets/coco/eval_vqa.yaml

@@ -0,0 +1,27 @@
+ # Copyright (c) 2022, salesforce.com, inc.
+ # All rights reserved.
+ # SPDX-License-Identifier: BSD-3-Clause
+ # For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+
+datasets:
+  coco_vqa:
+    # data_dir: ${env.data_dir}/datasets
+    data_type: images # [images|videos|features]
+
+    build_info:
+      # Be careful not to append minus sign (-) before split to avoid itemizing
+      annotations:
+        val:
+          url:
+              # TODO make this order insensitive
+              - https://storage.googleapis.com/sfr-vision-language-research/LAVIS/datasets/vqav2/vqa_val_eval.json
+              - https://storage.googleapis.com/sfr-vision-language-research/LAVIS/datasets/vqav2/answer_list.json
+              - https://storage.googleapis.com/sfr-vision-language-research/LAVIS/datasets/vqav2/v2_OpenEnded_mscoco_val2014_questions.json
+              - https://storage.googleapis.com/sfr-vision-language-research/LAVIS/datasets/vqav2/v2_mscoco_val2014_annotations.json
+          storage:
+              - coco/annotations/vqa_val_eval.json
+              - coco/annotations/answer_list.json
+              - coco/annotations/v2_OpenEnded_mscoco_val2014_questions.json
+              - coco/annotations/v2_mscoco_val2014_annotations.json
+      images:
+          storage: coco/images/

LAVIS-main/lavis/configs/datasets/coin/defaults_cap.yaml

@@ -0,0 +1,51 @@
+ # Copyright (c) 2023, salesforce.com, inc.
+ # All rights reserved.
+ # SPDX-License-Identifier: BSD-3-Clause
+ # For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+
+datasets:
+  coin_caption: # name of the dataset builder
+    # data_dir: ${env.data_dir}/datasets
+    data_type: videos # [images|videos|features]
+
+    vis_processor:
+        train:
+          name: alpro_video_train
+          n_frms: 4
+          image_size: 224
+          min_scale: 0.9
+          max_scale: 1.0
+          full_video: False
+        eval:
+          name: alpro_video_eval
+          n_frms: 4
+          image_size: 224
+          min_scale: 0.9
+          max_scale: 1.0
+          full_video: False
+      
+    text_processor:
+        train:
+          name: blip_caption
+        eval:
+          name: blip_caption
+
+    build_info:
+      # Be careful not to append minus sign (-) before split to avoid itemizing
+      annotations:
+        train:
+          url: 
+            - https://storage.googleapis.com/sfr-xinstructblip-data-research/data/coin/train.json
+            # - /export/video-language-dataset/data/coin/annotations/train_lavis.json
+          storage: 
+            - coin/annotations/train.json
+            # - /export/video-language-dataset/data/coin/annotations/train_lavis.json
+        val:
+          url: 
+            - https://storage.googleapis.com/sfr-xinstructblip-data-research/data/coin/val.json
+            # - /export/video-language-dataset/data/coin/annotations/val_lavis.json
+          storage: 
+            - coin/annotations/val.json
+            # - /export/video-language-dataset/data/coin/annotations/val_lavis.json
+      videos:
+        storage: /export/video-language-dataset/data/coin/annotations/videos/

LAVIS-main/lavis/configs/datasets/coin/defaults_cap_instruct.yaml

@@ -0,0 +1,53 @@
+ # Copyright (c) 2023, salesforce.com, inc.
+ # All rights reserved.
+ # SPDX-License-Identifier: BSD-3-Clause
+ # For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+
+datasets:
+  coin_caption_instruct: # name of the dataset builder
+    # data_dir: ${env.data_dir}/datasets
+    data_type: videos # [images|videos|features]
+
+    vis_processor:
+        train:
+          name: alpro_video_train
+          n_frms: 4
+          image_size: 224
+          min_scale: 0.9
+          max_scale: 1.0
+          full_video: False
+        eval:
+          name: alpro_video_eval
+          n_frms: 4
+          image_size: 224
+          min_scale: 0.9
+          max_scale: 1.0
+          full_video: False
+      
+    text_processor:
+        train:
+          name: blip_instruction
+          task: caption
+          modality: image
+        eval:
+          name: blip_caption
+
+    build_info:
+      # Be careful not to append minus sign (-) before split to avoid itemizing
+      annotations:
+        train:
+          url: 
+            - https://storage.googleapis.com/sfr-xinstructblip-data-research/data/coin/train.json
+            # - /export/video-language-dataset/data/coin/annotations/train_lavis.json
+          storage: 
+            - coin/annotations/train.json
+            # - /export/video-language-dataset/data/coin/annotations/train_lavis.json
+        val:
+          url: 
+            - https://storage.googleapis.com/sfr-xinstructblip-data-research/data/coin/val.json
+            # - /export/video-language-dataset/data/coin/annotations/val_lavis.json
+          storage: 
+            - coin/annotations/val.json
+            # - /export/video-language-dataset/data/coin/annotations/val_lavis.json
+      videos:
+        storage: /export/video-language-dataset/data/coin/annotations/videos/

LAVIS-main/lavis/configs/datasets/conceptual_caption/defaults_12m.yaml

@@ -0,0 +1,20 @@
+ # Copyright (c) 2022, salesforce.com, inc.
+ # All rights reserved.
+ # SPDX-License-Identifier: BSD-3-Clause
+ # For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+
+datasets:
+  conceptual_caption_12m:
+    # data_dir: ${env.data_dir}/datasets
+    data_type: images # [images|videos|features]
+
+    build_info:
+      # Be careful not to append minus sign (-) before split to avoid itemizing
+      annotations:
+        train:
+          url:
+              - /export/home/workspace/datasets/cc12m.json
+          storage:
+              - conceptual_caption/annotations/cc12m.json
+      images:
+          storage: conceptual_caption/images_12m

LAVIS-main/lavis/configs/datasets/conceptual_caption/defaults_12m_instruct.yaml

@@ -0,0 +1,37 @@
+ # Copyright (c) 2023, salesforce.com, inc.
+ # All rights reserved.
+ # SPDX-License-Identifier: BSD-3-Clause
+ # For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+
+datasets:
+  conceptual_caption_12m_instruct:
+    # data_dir: ${env.data_dir}/datasets
+    data_type: images # [images|videos|features]
+
+    vis_processor:
+      train:
+        name: "clip_image_train"
+        image_size: 224
+      eval:
+        name: "clip_image_eval"
+        image_size: 224
+      
+    text_processor:
+      train:
+        name: blip_instruction
+        task: caption
+        modality: image
+      eval:
+        name: blip_caption
+        
+    build_info:
+      # Be careful not to append minus sign (-) before split to avoid itemizing
+      annotations:
+        train:
+          url:
+            - https://storage.googleapis.com/sfr-xinstructblip-data-research/data/cc12m/x_instructblip_clean.json
+              # - /export/home/workspace/datasets/cc12m.json
+          storage:
+              - conceptual_caption/annotations/cc12m.json
+      images:
+          storage: conceptual_caption/images_12m

LAVIS-main/lavis/configs/datasets/conceptual_caption/defaults_3m.yaml

@@ -0,0 +1,20 @@
+ # Copyright (c) 2022, salesforce.com, inc.
+ # All rights reserved.
+ # SPDX-License-Identifier: BSD-3-Clause
+ # For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+
+datasets:
+  conceptual_caption_3m:
+    # data_dir: ${env.data_dir}/datasets
+    data_type: images # [images|videos|features]
+
+    build_info:
+      # Be careful not to append minus sign (-) before split to avoid itemizing
+      annotations:
+        train:
+          url:
+              - /export/home/workspace/datasets/cc3m.json
+          storage:
+              - conceptual_caption/annotations/cc3m.json
+      images:
+          storage: conceptual_caption/images

LAVIS-main/lavis/configs/datasets/conceptual_caption/defaults_3m_instruct.yaml

@@ -0,0 +1,36 @@
+ # Copyright (c) 2023, salesforce.com, inc.
+ # All rights reserved.
+ # SPDX-License-Identifier: BSD-3-Clause
+ # For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+
+datasets:
+  conceptual_caption_3m_instruct:
+    # data_dir: ${env.data_dir}/datasets
+    data_type: images # [images|videos|features]
+
+    vis_processor:
+      train:
+        name: "clip_image_train"
+        image_size: 224
+      eval:
+        name: "clip_image_eval"
+        image_size: 224
+    
+    text_processor:
+      train:
+        name: blip_instruction
+        task: caption
+        modality: image
+      eval:
+        name: blip_caption
+
+    build_info:
+      # Be careful not to append minus sign (-) before split to avoid itemizing
+      annotations:
+        train:
+          url:
+              - /export/home/workspace/datasets/cc3m.json
+          storage:
+              - conceptual_caption/annotations/cc3m.json
+      images:
+          storage: conceptual_caption/images

LAVIS-main/lavis/configs/datasets/didemo/defaults_ret.yaml

@@ -0,0 +1,25 @@
+ # Copyright (c) 2022, salesforce.com, inc.
+ # All rights reserved.
+ # SPDX-License-Identifier: BSD-3-Clause
+ # For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+
+datasets:
+  didemo_retrieval: # name of the dataset builder
+    # data_dir: ${env.data_dir}/datasets
+    data_type: videos # [images|videos|features]
+
+    build_info:
+      # Be careful not to append minus sign (-) before split to avoid itemizing
+      annotations:
+        train:
+          url: https://storage.googleapis.com/sfr-vision-language-research/LAVIS/datasets/didemo/retrieval_train.json
+          storage: didemo/annotations/retrieval_train.json
+        val:
+          url: https://storage.googleapis.com/sfr-vision-language-research/LAVIS/datasets/didemo/retrieval_val.json
+          storage: didemo/annotations/retrieval_val.json
+        test:
+          url: https://storage.googleapis.com/sfr-vision-language-research/LAVIS/datasets/didemo/retrieval_test.json
+          storage: didemo/annotations/retrieval_test.json
+      videos:
+        storage: didemo/videos
+        # storage: /export/share/dongxuli/data/didemo_retrieval/videos