Update configs/kosmos_ph_calvin_abcd.json

configs/kosmos_ph_calvin_abcd.json

@@ -6,16 +6,8 @@
     "model_url": "https://huggingface.co/microsoft/kosmos-2-patch14-224",
     "seq_len": 1,
     "image_size": 224,
-    "image_mean": [
-        0.48145466,
-        0.4578275,
-        0.40821073
-    ],
-    "image_std": [
-        0.26862954,
-        0.26130258,
-        0.27577711
-    ],
+    "image_mean": [0.48145466, 0.4578275, 0.40821073],
+    "image_std": [0.26862954, 0.26130258, 0.27577711],
     "window_size": 16,
     "fwd_pred_next_n": 10,
     "arm_gripper_loss_ratio": 0.01,
@@ -26,8 +18,8 @@
     "num_workers": 16,
     "data_scale": 1,
     "optimizer": "adam",
-    "learning_rate": 0.0001,
-    "min_lr_scale": 0.01,
+    "learning_rate": 2e-5,
+    "min_lr_scale": 1e-2,
     "weight_decay": 0,
     "warmup_epochs": 0.25,
     "warmup_steps": 0,
@@ -40,16 +32,16 @@
     "use_vision_resampler": false,
     "vision_masked_ratio": 0.9,
     "use_tube_mask": false,
-    "output_root": "checkpoints/kosmos/calvin_finetune",
-    "log_root": "logs/kosmos/calvin_finetune",
-    "cache_root": "cache/kosmos",
+    "output_root": "runs/checkpoints",
+    "log_root": "runs/logs",
+    "cache_root": "runs/cache",
     "model_load_path": null,
     "model_load_source": "torch",
     "resume": null,
-    "model_path": "/mnt/bn/robotics-data-lxh-lq/LLaVA/kosmos-2-patch14-224",
-    "model_config": "/mnt/bn/robotics-data-lxh-lq/LLaVA/kosmos-2-patch14-224/config.json",
+    "model_path": ".vlms/kosmos-2-patch14-224",
+    "model_config": ".vlms/kosmos-2-patch14-224/config.json",
     "train_setup": {
-        "precision": "16",
+        "precision": "bf16",
         "predict_action": true,
         "predict_forward": false,
         "predict_forward_hand": false,
@@ -64,7 +56,7 @@
         "mm_use_im_patch_token": false,
         "gradient_checkpointing": false,
         "lora_enable": false,
-        "mm_projector_lr": 0.0001,
+        "mm_projector_lr": 1e-4,
         "lora_r": 64,
         "lora_alpha": 16,
         "lora_dropout": 0.05,
@@ -94,18 +86,21 @@
     "fwd_head": null,
     "tokenizer": {
         "type": "AutoProcessor",
-        "pretrained_model_name_or_path": "/mnt/bn/robotics-data-lxh-lq/LLaVA/kosmos-2-patch14-224",
+        "pretrained_model_name_or_path": ".vlms/kosmos-2-patch14-224",
         "tokenizer_type": "kosmos",
         "max_text_len": 256,
         "additional_special_tokens": null
     },
+    "vlm": {
+        "type": "AutoModelForVision2Seq",
+        "name": "kosmos",
+        "pretrained_model_name_or_path": ".vlms/kosmos-2-patch14-224"
+    },
     "trainer": {
         "accelerator": "gpu",
         "strategy": "deepspeed_stage_2",
         "precision": "16",
-        "logger": [
-            "tensorboard"
-        ],
+        "logger": ["tensorboard"],
         "gradient_clip_val": 1.0,
         "use_distributed_sampler": false,
         "log_every_n_steps": 10,
@@ -117,7 +112,7 @@
     },
     "train_dataset": {
         "type": "DiskCalvinDataset",
-        "data_dir": "/mnt/bn/robotics/manipulation_data/calvin_data/task_ABCD_D/training",
+        "data_dir": "datasets/calvin_data/task_ABCD_D/training",
         "shift_first": false,
         "model_name": "kosmos",
         "rgb_pad": 10,
@@ -125,31 +120,10 @@
     },
     "val_dataset": {
         "type": "DiskCalvinDataset",
-        "data_dir": "/mnt/bn/robotics/manipulation_data/calvin_data/task_ABCD_D/validation",
+        "data_dir": "datasets/calvin_data/task_ABCD_D/validation",
         "model_name": "kosmos"
     },
     "norm_action": true,
     "norm_min": -0.65,
-    "norm_max": 0.65,
-    "scheduler": "cosine",
-    "raw_config_path": "configs/kosmos/finetune_kosmos_cont-lstm-post_full-ft_text_vision_wd=0_hist=16_act=10_use-hand_aug-shift_act-norm_lr-1e-4.json",
-    "config": "configs/kosmos/finetune_kosmos_cont-lstm-post_full-ft_text_vision_wd=0_hist=16_act=10_use-hand_aug-shift_act-norm_lr-1e-4.json",
-    "gpus": 8,
-    "num_nodes": 4,
-    "log_dir": "logs/kosmos/calvin_finetune/2024-11-20/04-00",
-    "output_dir": "checkpoints/kosmos/calvin_finetune/2024-11-20/04-00",
-    "data_dir": null,
-    "annotation_file": null,
-    "data_subfolder": null,
-    "task_num": null,
-    "exp_name": "17-54",
-    "use_multi_modal_emb": false,
-    "no_video_pretrained_model": false,
-    "finetune": false,
-    "llm": {
-        "type": null,
-        "n_embd": null,
-        "n_layer": null,
-        "n_head": null
-    }
+    "norm_max": 0.65
 }