Update configs/kosmos_ph_oxe-pretrain.json

configs/kosmos_ph_oxe-pretrain.json

@@ -1,18 +1,13 @@
 {
+    "robovlm_name": "RoboKosMos",
     "parent": null,
+    "task_name": "calvin_finetune",
     "model": "kosmos",
+    "model_url": "https://huggingface.co/microsoft/kosmos-2-patch14-224",
     "seq_len": 1,
     "image_size": 224,
-    "image_mean": [
-        0.48145466,
-        0.4578275,
-        0.40821073
-    ],
-    "image_std": [
-        0.26862954,
-        0.26130258,
-        0.27577711
-    ],
+    "image_mean": [0.48145466, 0.4578275, 0.40821073],
+    "image_std": [0.26862954, 0.26130258, 0.27577711],
     "window_size": 16,
     "fwd_pred_next_n": 10,
     "arm_gripper_loss_ratio": 0.01,
@@ -23,11 +18,11 @@
     "num_workers": 16,
     "data_scale": 1,
     "optimizer": "adam",
-    "learning_rate": 2e-05,
-    "min_lr_scale": 0.01,
+    "learning_rate": 2e-5,
+    "min_lr_scale": 1e-2,
     "weight_decay": 0,
-    "warmup_epochs": 0,
-    "warmup_steps": 5000,
+    "warmup_epochs": 0.25,
+    "warmup_steps": 0,
     "warmup_ratio": null,
     "use_hand_rgb": true,
     "use_time_causal_attn": false,
@@ -37,16 +32,16 @@
     "use_vision_resampler": false,
     "vision_masked_ratio": 0.9,
     "use_tube_mask": false,
-    "output_root": "/mnt/bn/robotics-data-lxh-lq-v2/checkpoints/video_pretrain_manipulation/kosmos/calvin_finetune",
-    "log_root": "/mnt/bn/robotics-data-lxh-lq-v2/logs/video_pretrain_manipulation/kosmos/calvin_finetune",
-    "cache_root": "/mnt/bn/robotics-data-lxh-lq-v2/cache/video_pretrain_manipulation/kosmos",
+    "output_root": "runs/checkpoints",
+    "log_root": "runs/logs",
+    "cache_root": "runs/cache",
     "model_load_path": null,
     "model_load_source": "torch",
     "resume": null,
-    "model_path": "/mnt/bn/robotics-data-lxh-lq/LLaVA/kosmos-2-patch14-224",
-    "model_config": "/mnt/bn/robotics-data-lxh-lq/LLaVA/kosmos-2-patch14-224/config.json",
+    "model_path": ".vlms/kosmos-2-patch14-224",
+    "model_config": ".vlms/kosmos-2-patch14-224/config.json",
     "train_setup": {
-        "precision": "16-mixed",
+        "precision": "bf16",
         "predict_action": true,
         "predict_forward": false,
         "predict_forward_hand": false,
@@ -61,7 +56,7 @@
         "mm_use_im_patch_token": false,
         "gradient_checkpointing": false,
         "lora_enable": false,
-        "mm_projector_lr": 0.0001,
+        "mm_projector_lr": 1e-4,
         "lora_r": 64,
         "lora_alpha": 16,
         "lora_dropout": 0.05,
@@ -91,31 +86,33 @@
     "fwd_head": null,
     "tokenizer": {
         "type": "AutoProcessor",
-        "pretrained_model_name_or_path": "/mnt/bn/robotics-data-lxh-lq/LLaVA/kosmos-2-patch14-224",
+        "pretrained_model_name_or_path": ".vlms/kosmos-2-patch14-224",
         "tokenizer_type": "kosmos",
         "max_text_len": 256,
         "additional_special_tokens": null
     },
+    "vlm": {
+        "type": "AutoModelForVision2Seq",
+        "name": "kosmos",
+        "pretrained_model_name_or_path": ".vlms/kosmos-2-patch14-224"
+    },
     "trainer": {
         "accelerator": "gpu",
         "strategy": "deepspeed_stage_2",
-        "precision": "16-mixed",
-        "logger": [
-            "tensorboard"
-        ],
+        "precision": "16",
+        "logger": ["tensorboard"],
         "gradient_clip_val": 1.0,
         "use_distributed_sampler": false,
         "log_every_n_steps": 10,
         "max_epochs": 5,
         "val_check_interval": 10000,
         "check_val_every_n_epoch": null,
-        "max_steps": 100000,
-        "accumulate_grad_batches": 1,
-        "limit_val_batches": 1000
+        "max_steps": 50000,
+        "accumulate_grad_batches": 1
     },
     "train_dataset": {
         "type": "OpenVLADataset",
-        "data_root_dir": "/mnt/bn/robotics-data-lxh-lq/openvla/datasets/open-x-embodiment",
+        "data_root_dir": "datasets/open-x-embodiment",
         "model_name": "kosmos",
         "image_aug": true,
         "mode": "train",
@@ -127,7 +124,7 @@
     },
     "val_dataset": {
         "type": "OpenVLADataset",
-        "data_root_dir": "/mnt/bn/robotics-data-lxh-lq/openvla/datasets/open-x-embodiment",
+        "data_root_dir": "datasets/open-x-embodiment",
         "model_name": "kosmos",
         "mode": "train",
         "data_mix": "rt_1",
@@ -136,24 +133,7 @@
         "shuffle_buffer_size": 10000,
         "train": false
     },
-    "raw_config_path": "configs/kosmos/oxe_new/finetune_kosmos_cont-lstm-post_full-ft_text_vision_wd=0_hist=16_act=10_use-hand_aug-shift_act-norm_lr-2e-5_oxe.json",
-    "config": "configs/kosmos/oxe_new/finetune_kosmos_cont-lstm-post_full-ft_text_vision_wd=0_hist=16_act=10_use-hand_aug-shift_act-norm_lr-2e-5_oxe.json",
-    "gpus": 8,
-    "num_nodes": 4,
-    "log_dir": "/mnt/bn/robotics-data-lxh-lq-v2/logs/video_pretrain_manipulation/kosmos/calvin_finetune/2024-11-21/01-16",
-    "output_dir": "/mnt/bn/robotics-data-lxh-lq-v2/checkpoints/video_pretrain_manipulation/kosmos/calvin_finetune/2024-11-21/01-16",
-    "data_dir": null,
-    "annotation_file": null,
-    "data_subfolder": null,
-    "task_num": null,
-    "exp_name": "01-16",
-    "use_multi_modal_emb": false,
-    "no_video_pretrained_model": false,
-    "finetune": false,
-    "llm": {
-        "type": null,
-        "n_embd": null,
-        "n_layer": null,
-        "n_head": null
-    }
+    "norm_action": true,
+    "norm_min": -0.65,
+    "norm_max": 0.65
 }