Upload pi-zero.json

pi-zero.json

@@ -0,0 +1,269 @@
+{
+    "robovlm_name": "RoboPiZero",
+    "parent": null,
+    "task_name": "pizero_finetune",
+    "model": "pizero",
+    "model_url": "https://huggingface.co/google/paligemma-3b-pt-224",
+    "seq_len": 1,
+    "image_size": 224,
+    "image_mean": [
+        0.48145466,
+        0.4578275,
+        0.40821073
+    ],
+    "image_std": [
+        0.26862954,
+        0.26130258,
+        0.27577711
+    ],
+    "window_size": 1,
+    "fwd_pred_next_n": 4,
+    "arm_gripper_loss_ratio": 0.01,
+    "text_loss_ratio": 0.1,
+    "cap_loss_ratio": 0.05,
+    "fwd_loss_ratio": 0,
+    "vl_cotrain_ratio": 1,
+    "seed": 123,
+    "batch_size": 4,
+    "num_workers": 16,
+    "data_scale": 1,
+    "optimizer": "adam",
+    "learning_rate": 2e-05,
+    "min_lr_scale": 0.01,
+    "weight_decay": 0,
+    "warmup_epochs": 0.25,
+    "warmup_steps": 0,
+    "warmup_ratio": null,
+    "use_hand_rgb": true,
+    "use_time_causal_attn": false,
+    "use_mim_obs_loss": false,
+    "use_pixel_loss": true,
+    "use_obs_queries": true,
+    "use_vision_resampler": false,
+    "vision_masked_ratio": 0.9,
+    "use_tube_mask": false,
+    "cache_root": "runs/cache/pizero",
+    "model_load_path": null,
+    "model_load_source": "torch",
+    "resume": null,
+    "use_cot_stage_token": false,
+    "use_cot": false,
+    "force_model_cot": false,
+    "use_chat_stage_token": false,
+    "cot_tags": [
+        "task",
+        "plan",
+        "bboxes",
+        "subtask_reason",
+        "subtask",
+        "move_reason",
+        "move",
+        "gripper"
+    ],
+    "model_path": "/share/project/lxh/project/VLMs/paligemma-3b-pt-224",
+    "model_config": "/share/project/lxh/project/VLMs/paligemma-3b-pt-224",
+    "deepspeed_config": {
+        "fp16": {
+            "enabled": false,
+            "loss_scale": 0,
+            "loss_scale_window": 1000,
+            "initial_scale_power": 16,
+            "hysteresis": 2,
+            "min_loss_scale": 1
+        },
+        "bf16": {
+            "enabled": true
+        },
+        "zero_allow_untested_optimizer": true,
+        "train_micro_batch_size_per_gpu": 4,
+        "zero_optimization": {
+            "stage": 2,
+            "overlap_comm": true,
+            "contiguous_gradients": true,
+            "reduce_bucket_size": 500000000.0,
+            "allgather_partitions": true,
+            "allgather_bucket_size": 500000000.0,
+            "reduce_scatter": true
+        },
+        "communication": {
+            "nccl_timeout": 3600,
+            "all_reduce_timeout": 7200
+        },
+        "gradient_accumulation_steps": 4,
+        "gradient_clipping": 0.0
+    },
+    "train_setup": {
+        "precision": "bf16",
+        "predict_action": true,
+        "predict_forward": false,
+        "predict_forward_hand": false,
+        "predict_caption": false,
+        "train_vision": true,
+        "bits": -1,
+        "freeze_mm_mlp_adapter": false,
+        "freeze_backbone": false,
+        "freeze_resampler": false,
+        "tune_mm_mlp_adapter": false,
+        "mm_use_im_start_end": false,
+        "mm_use_im_patch_token": false,
+        "gradient_checkpointing": false,
+        "lora_enable": false,
+        "mm_projector_lr": 0.0001,
+        "lora_r": 64,
+        "lora_alpha": 16,
+        "lora_dropout": 0.05,
+        "lora_bias": "none",
+        "train_text_embedding": true
+    },
+    "vision_resampler": {
+        "vis_dim": 1024,
+        "depth": 8,
+        "dim_head": 64,
+        "heads": 8,
+        "num_latents": 64
+    },
+    "act_encoder": null,
+    "act_head": {
+        "type": "FCDecoder",
+        "hidden_size": 1024,
+        "action_dim": 7,
+        "down_sample": "none",
+        "latent": 1,
+        "fwd_pred_next_n": 1,
+        "window_size": 1,
+        "action_space": "continuous",
+        "with_history": true,
+        "history_type": "post",
+        "n_bin": 256,
+        "min_action": -1,
+        "max_action": 1
+    },
+    "fwd_head": null,
+    "tokenizer": {
+        "type": "AutoProcessor",
+        "pretrained_model_name_or_path": "/share/project/lxh/project/VLMs/paligemma-3b-pt-224",
+        "tokenizer_type": "paligemma",
+        "max_text_len": 256,
+        "additional_special_tokens": null
+    },
+    "vlm": {
+        "type": "Pizero",
+        "name": "pizero",
+        "dtype": "bfloat16",
+        "pretrained_model_name_or_path": "/share/project/lxh/project/VLMs/paligemma-3b-pt-224",
+        "mixture": {
+            "vlm": {
+                "hidden_size": 2048,
+                "intermediate_size": 16384,
+                "use_final_norm": false,
+                "cache": true,
+                "rope_theta": 10000.0,
+                "use_quantize": false,
+                "use_lora": false,
+                "adaptive_mode": null
+            },
+            "proprio": {
+                "hidden_size": 1024,
+                "intermediate_size": 4096,
+                "use_final_norm": true,
+                "cache": true,
+                "rope_theta": 100.0,
+                "use_quantize": false,
+                "use_lora": false,
+                "adaptive_mode": null
+            },
+            "action": {
+                "hidden_size": 1024,
+                "intermediate_size": 4096,
+                "use_final_norm": true,
+                "cache": false,
+                "rope_theta": 100.0,
+                "use_quantize": false,
+                "use_lora": false,
+                "adaptive_mode": null
+            }
+        },
+        "num_hidden_layers": 18,
+        "num_attention_heads": 8,
+        "num_key_value_heads": 1,
+        "num_inference_steps": 10,
+        "head_dim": 256,
+        "rms_norm_eps": 1e-06,
+        "attention_bias": false,
+        "attention_dropout": 0.0,
+        "max_image_text_tokens": 276,
+        "cond_steps": 1,
+        "horizon_steps": 4,
+        "pad_token_id": 0,
+        "time_hidden_size": 256,
+        "action_expert_adaptive_mode": null,
+        "final_action_clip_value": 1.0,
+        "action_dim": 7,
+        "proprio_dim": 7,
+        "time_max_period": 100.0,
+        "flow_sampling": "beta",
+        "flow_alpha": 1.5,
+        "flow_beta": 1.0,
+        "flow_sig_min": 0.001
+    },
+    "trainer": {
+        "accelerator": "gpu",
+        "strategy": "deepspeed_stage_2",
+        "precision": "bf16",
+        "logger": [
+            "tensorboard"
+        ],
+        "gradient_clip_val": 1.0,
+        "use_distributed_sampler": false,
+        "log_every_n_steps": 10,
+        "max_epochs": 10,
+        "val_check_interval": null,
+        "check_val_every_n_epoch": 1,
+        "max_steps": -1,
+        "accumulate_grad_batches": 4
+    },
+    "use_cot_data": true,
+    "train_dataset": {
+        "type": "OpenVLADatasetByRank",
+        "data_root_dir": "/share/project/lxh/datasets/tf_datasets",
+        "model_name": "paligemma",
+        "image_aug": true,
+        "mode": "train",
+        "data_mix": "bridge",
+        "window_sample": "sliding",
+        "organize_type": "interleave",
+        "shuffle_buffer_size": 51200,
+        "train": true
+    },
+    "val_dataset": {
+        "type": "OpenVLADatasetByRank",
+        "data_root_dir": "/share/project/lxh/datasets/tf_datasets",
+        "model_name": "paligemma",
+        "mode": "train",
+        "data_mix": "bridge",
+        "window_sample": "sliding",
+        "organize_type": "interleave",
+        "shuffle_buffer_size": 10000,
+        "train": false
+    },
+    "raw_config_path": "/share/project/lxh/project/RoboVLMs-dev/configs/baai_oxe/finetune_pizero_bridge_no-cot.json",
+    "config": "/share/project/lxh/project/RoboVLMs-dev/configs/baai_oxe/finetune_pizero_bridge_no-cot.json",
+    "gpus": 8,
+    "num_nodes": 4,
+    "log_dir": "runs/logs/pizero/pizero_finetune/2025-04-29/05-54",
+    "output_dir": "runs/checkpoints/pizero/pizero_finetune/2025-04-29/05-54",
+    "data_dir": null,
+    "annotation_file": null,
+    "data_subfolder": null,
+    "task_num": null,
+    "exp_name": "05-54",
+    "use_multi_modal_emb": false,
+    "no_video_pretrained_model": false,
+    "finetune": false,
+    "llm": {
+        "type": null,
+        "n_embd": null,
+        "n_layer": null,
+        "n_head": null
+    }
+}