diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..50928770a50099af45d7098f6293f04572400c0a 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +wandb/run-20260408_061152-ooipb6on/run-ooipb6on.wandb filter=lfs diff=lfs merge=lfs -text diff --git a/checkpoints/000500/pretrained_model/config.json b/checkpoints/000500/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..3bdae2d4dfa4b23cad255243a126b98d8f5d62c6 --- /dev/null +++ b/checkpoints/000500/pretrained_model/config.json @@ -0,0 +1,111 @@ +{ + "type": "smolvla", + "n_obs_steps": 1, + "input_features": { + "observation.images.image": { + "type": "VISUAL", + "shape": [ + 256, + 256, + 3 + ] + }, + "observation.images.wrist_image": { + "type": "VISUAL", + "shape": [ + 256, + 256, + 3 + ] + }, + "observation.state": { + "type": "STATE", + "shape": [ + 8 + ] + }, + "observation.states.ee_state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.states.joint_state": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "observation.states.gripper_state": { + "type": "STATE", + "shape": [ + 2 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "device": "cuda", + "use_amp": false, + "use_peft": false, + "push_to_hub": true, + "repo_id": "ardalon/sequential_t1_task48_fft", + "private": null, + "tags": null, + "license": null, + "pretrained_path": "/content/smolvla_base", + "chunk_size": 50, + "n_action_steps": 50, + "normalization_mapping": { + "VISUAL": "IDENTITY", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "max_state_dim": 32, + "max_action_dim": 32, + "resize_imgs_with_padding": [ + 512, + 512 + ], + "empty_cameras": 0, + "adapt_to_pi_aloha": false, + "use_delta_joint_actions_aloha": false, + "tokenizer_max_length": 48, + "num_steps": 10, + "use_cache": true, + "freeze_vision_encoder": true, + "train_expert_only": true, + "train_state_proj": true, + "optimizer_lr": 2e-05, + "optimizer_betas": [ + 0.9, + 0.95 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 0.01, + "optimizer_grad_clip_norm": 10.0, + "scheduler_warmup_steps": 1000, + "scheduler_decay_steps": 30000, + "scheduler_decay_lr": 2.5e-06, + "vlm_model_name": "HuggingFaceTB/SmolVLM2-500M-Video-Instruct", + "load_vlm_weights": true, + "add_image_special_tokens": false, + "attention_mode": "cross_attn", + "prefix_length": 0, + "pad_language_to": "max_length", + "num_expert_layers": 0, + "num_vlm_layers": 16, + "self_attn_every_n_layers": 2, + "expert_width_multiplier": 0.75, + "min_period": 0.004, + "max_period": 4.0, + "rtc_config": null, + "compile_model": false, + "compile_mode": "max-autotune" +} \ No newline at end of file diff --git a/checkpoints/000500/pretrained_model/model.safetensors b/checkpoints/000500/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4f6a7b358f9e1a6fadda1e900d8d18d862c915c1 --- /dev/null +++ b/checkpoints/000500/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3f97600ff7675ca5232f463285f542694bc0e2487011ffcebee60be9309f455 +size 906712520 diff --git a/checkpoints/000500/pretrained_model/policy_postprocessor.json b/checkpoints/000500/pretrained_model/policy_postprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..0ec8146a07f9b92bc5a3c02e2d6b7d49ba6d26e8 --- /dev/null +++ b/checkpoints/000500/pretrained_model/policy_postprocessor.json @@ -0,0 +1,32 @@ +{ + "name": "policy_postprocessor", + "steps": [ + { + "registry_name": "unnormalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "norm_map": { + "VISUAL": "IDENTITY", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + } + }, + "state_file": "policy_postprocessor_step_0_unnormalizer_processor.safetensors" + }, + { + "registry_name": "device_processor", + "config": { + "device": "cpu", + "float_dtype": null + } + } + ] +} \ No newline at end of file diff --git a/checkpoints/000500/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors b/checkpoints/000500/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..fb3b6bb870bf669cd89d6bc4600478c768d22459 --- /dev/null +++ b/checkpoints/000500/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2abc5a6a77ada2e972696d0b4da9485e8ac2c2036629207a51cf587b2c434abb +size 5440 diff --git a/checkpoints/000500/pretrained_model/policy_preprocessor.json b/checkpoints/000500/pretrained_model/policy_preprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..7527b68cb0a2d7c7a1a0f858ac44bec3df5ac0bf --- /dev/null +++ b/checkpoints/000500/pretrained_model/policy_preprocessor.json @@ -0,0 +1,97 @@ +{ + "name": "policy_preprocessor", + "steps": [ + { + "registry_name": "rename_observations_processor", + "config": { + "rename_map": {} + } + }, + { + "registry_name": "to_batch_processor", + "config": {} + }, + { + "registry_name": "smolvla_new_line_processor", + "config": {} + }, + { + "registry_name": "tokenizer_processor", + "config": { + "max_length": 48, + "task_key": "task", + "padding_side": "right", + "padding": "max_length", + "truncation": true, + "tokenizer_name": "HuggingFaceTB/SmolVLM2-500M-Video-Instruct" + } + }, + { + "registry_name": "device_processor", + "config": { + "device": "cuda", + "float_dtype": null + } + }, + { + "registry_name": "normalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "observation.images.image": { + "type": "VISUAL", + "shape": [ + 256, + 256, + 3 + ] + }, + "observation.images.wrist_image": { + "type": "VISUAL", + "shape": [ + 256, + 256, + 3 + ] + }, + "observation.state": { + "type": "STATE", + "shape": [ + 8 + ] + }, + "observation.states.ee_state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.states.joint_state": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "observation.states.gripper_state": { + "type": "STATE", + "shape": [ + 2 + ] + }, + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "norm_map": { + "VISUAL": "IDENTITY", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + } + }, + "state_file": "policy_preprocessor_step_5_normalizer_processor.safetensors" + } + ] +} \ No newline at end of file diff --git a/checkpoints/000500/pretrained_model/policy_preprocessor_step_5_normalizer_processor.safetensors b/checkpoints/000500/pretrained_model/policy_preprocessor_step_5_normalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..fb3b6bb870bf669cd89d6bc4600478c768d22459 --- /dev/null +++ b/checkpoints/000500/pretrained_model/policy_preprocessor_step_5_normalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2abc5a6a77ada2e972696d0b4da9485e8ac2c2036629207a51cf587b2c434abb +size 5440 diff --git a/checkpoints/000500/pretrained_model/train_config.json b/checkpoints/000500/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..06c696e54ad5d01142c45fe44d4d30291256fb50 --- /dev/null +++ b/checkpoints/000500/pretrained_model/train_config.json @@ -0,0 +1,392 @@ +{ + "dataset": { + "repo_id": "daniecraig/libero_90_no_noops_lerobot_v30", + "root": null, + "episodes": [ + 3092, + 3093, + 3094, + 3095, + 3096, + 3097, + 3098, + 3099, + 3100, + 3101, + 3102, + 3103, + 3104, + 3105, + 3106, + 3107, + 3108, + 3109, + 3110, + 3111, + 3112, + 3113, + 3114, + 3115, + 3116, + 3117, + 3118, + 3119, + 3120, + 3121, + 3122, + 3123, + 3124, + 3125, + 3126, + 3127, + 3128, + 3129, + 3130, + 3131, + 3132, + 3133, + 3134, + 3135, + 3136 + ], + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + }, + "affine": { + "weight": 1.0, + "type": "RandomAffine", + "kwargs": { + "degrees": [ + -5.0, + 5.0 + ], + "translate": [ + 0.05, + 0.05 + ] + } + } + } + }, + "revision": "main", + "use_imagenet_stats": true, + "video_backend": "torchcodec", + "streaming": false + }, + "env": { + "type": "libero", + "task": "libero_90", + "fps": 30, + "features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + }, + "pixels/agentview_image": { + "type": "VISUAL", + "shape": [ + 360, + 360, + 3 + ] + }, + "pixels/robot0_eye_in_hand_image": { + "type": "VISUAL", + "shape": [ + 360, + 360, + 3 + ] + }, + "robot_state/eef/pos": { + "type": "STATE", + "shape": [ + 3 + ] + }, + "robot_state/eef/quat": { + "type": "STATE", + "shape": [ + 4 + ] + }, + "robot_state/eef/mat": { + "type": "STATE", + "shape": [ + 3, + 3 + ] + }, + "robot_state/gripper/qpos": { + "type": "STATE", + "shape": [ + 2 + ] + }, + "robot_state/gripper/qvel": { + "type": "STATE", + "shape": [ + 2 + ] + }, + "robot_state/joints/pos": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "robot_state/joints/vel": { + "type": "STATE", + "shape": [ + 7 + ] + } + }, + "features_map": { + "action": "action", + "robot_state/eef/pos": "observation.state.eef_pos", + "robot_state/eef/quat": "observation.state.eef_quat", + "robot_state/eef/mat": "observation.state.eef_mat", + "robot_state/gripper/qpos": "observation.state.gripper_qpos", + "robot_state/gripper/qvel": "observation.state.gripper_qvel", + "robot_state/joints/pos": "observation.state.joint_pos", + "robot_state/joints/vel": "observation.state.joint_vel", + "pixels/agentview_image": "observation.images.image", + "pixels/robot0_eye_in_hand_image": "observation.images.image2" + }, + "max_parallel_tasks": 1, + "disable_env_checker": true, + "task_ids": [ + 48 + ], + "episode_length": null, + "obs_type": "pixels_agent_pos", + "render_mode": "rgb_array", + "camera_name": "agentview_image,robot0_eye_in_hand_image", + "init_states": true, + "camera_name_mapping": null, + "observation_height": 360, + "observation_width": 360, + "control_mode": "relative" + }, + "policy": { + "type": "smolvla", + "n_obs_steps": 1, + "input_features": { + "observation.images.image": { + "type": "VISUAL", + "shape": [ + 256, + 256, + 3 + ] + }, + "observation.images.wrist_image": { + "type": "VISUAL", + "shape": [ + 256, + 256, + 3 + ] + }, + "observation.state": { + "type": "STATE", + "shape": [ + 8 + ] + }, + "observation.states.ee_state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.states.joint_state": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "observation.states.gripper_state": { + "type": "STATE", + "shape": [ + 2 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "device": "cuda", + "use_amp": false, + "use_peft": false, + "push_to_hub": true, + "repo_id": "ardalon/sequential_t1_task48_fft", + "private": null, + "tags": null, + "license": null, + "pretrained_path": "/content/smolvla_base", + "chunk_size": 50, + "n_action_steps": 50, + "normalization_mapping": { + "VISUAL": "IDENTITY", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "max_state_dim": 32, + "max_action_dim": 32, + "resize_imgs_with_padding": [ + 512, + 512 + ], + "empty_cameras": 0, + "adapt_to_pi_aloha": false, + "use_delta_joint_actions_aloha": false, + "tokenizer_max_length": 48, + "num_steps": 10, + "use_cache": true, + "freeze_vision_encoder": true, + "train_expert_only": true, + "train_state_proj": true, + "optimizer_lr": 2e-05, + "optimizer_betas": [ + 0.9, + 0.95 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 0.01, + "optimizer_grad_clip_norm": 10.0, + "scheduler_warmup_steps": 1000, + "scheduler_decay_steps": 30000, + "scheduler_decay_lr": 2.5e-06, + "vlm_model_name": "HuggingFaceTB/SmolVLM2-500M-Video-Instruct", + "load_vlm_weights": true, + "add_image_special_tokens": false, + "attention_mode": "cross_attn", + "prefix_length": 0, + "pad_language_to": "max_length", + "num_expert_layers": 0, + "num_vlm_layers": 16, + "self_attn_every_n_layers": 2, + "expert_width_multiplier": 0.75, + "min_period": 0.004, + "max_period": 4.0, + "rtc_config": null, + "compile_model": false, + "compile_mode": "max-autotune" + }, + "output_dir": "outputs/train/sequential_t1_task48_fft", + "job_name": "sequential_t1_task48_fft", + "resume": false, + "seed": 42, + "cudnn_deterministic": false, + "num_workers": 4, + "batch_size": 16, + "steps": 3000, + "eval_freq": 0, + "log_freq": 1, + "tolerance_s": 0.0001, + "save_checkpoint": true, + "save_freq": 500, + "use_policy_training_preset": true, + "optimizer": { + "type": "adamw", + "lr": 2e-05, + "weight_decay": 0.01, + "grad_clip_norm": 10.0, + "betas": [ + 0.9, + 0.95 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "cosine_decay_with_warmup", + "num_warmup_steps": 1000, + "num_decay_steps": 30000, + "peak_lr": 2e-05, + "decay_lr": 2.5e-06 + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": true, + "project": "lerobot-smolvla", + "entity": null, + "notes": null, + "run_id": "ooipb6on", + "mode": null, + "add_tags": true + }, + "peft": null, + "use_rabc": false, + "rabc_progress_path": null, + "rabc_kappa": 0.01, + "rabc_epsilon": 1e-06, + "rabc_head_mode": "sparse", + "rename_map": {}, + "checkpoint_path": null +} \ No newline at end of file diff --git a/checkpoints/000500/training_state/optimizer_param_groups.json b/checkpoints/000500/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..946b4cbe4e5b4d4cc2e9b5bb3a1984bb926a3c33 --- /dev/null +++ b/checkpoints/000500/training_state/optimizer_param_groups.json @@ -0,0 +1,521 @@ +[ + { + "lr": 1.882772228311384e-05, + "betas": [ + 0.9, + 0.95 + ], + "eps": 1e-08, + "weight_decay": 0.01, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "initial_lr": 2e-05, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310, + 311, + 312, + 313, + 314, + 315, + 316, + 317, + 318, + 319, + 320, + 321, + 322, + 323, + 324, + 325, + 326, + 327, + 328, + 329, + 330, + 331, + 332, + 333, + 334, + 335, + 336, + 337, + 338, + 339, + 340, + 341, + 342, + 343, + 344, + 345, + 346, + 347, + 348, + 349, + 350, + 351, + 352, + 353, + 354, + 355, + 356, + 357, + 358, + 359, + 360, + 361, + 362, + 363, + 364, + 365, + 366, + 367, + 368, + 369, + 370, + 371, + 372, + 373, + 374, + 375, + 376, + 377, + 378, + 379, + 380, + 381, + 382, + 383, + 384, + 385, + 386, + 387, + 388, + 389, + 390, + 391, + 392, + 393, + 394, + 395, + 396, + 397, + 398, + 399, + 400, + 401, + 402, + 403, + 404, + 405, + 406, + 407, + 408, + 409, + 410, + 411, + 412, + 413, + 414, + 415, + 416, + 417, + 418, + 419, + 420, + 421, + 422, + 423, + 424, + 425, + 426, + 427, + 428, + 429, + 430, + 431, + 432, + 433, + 434, + 435, + 436, + 437, + 438, + 439, + 440, + 441, + 442, + 443, + 444, + 445, + 446, + 447, + 448, + 449, + 450, + 451, + 452, + 453, + 454, + 455, + 456, + 457, + 458, + 459, + 460, + 461, + 462, + 463, + 464, + 465, + 466, + 467, + 468, + 469, + 470, + 471, + 472, + 473, + 474, + 475, + 476, + 477, + 478, + 479, + 480, + 481, + 482, + 483, + 484, + 485, + 486, + 487, + 488, + 489, + 490, + 491, + 492, + 493, + 494, + 495, + 496, + 497, + 498, + 499 + ] + } +] \ No newline at end of file diff --git a/checkpoints/000500/training_state/optimizer_state.safetensors b/checkpoints/000500/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..27088670c9e55a1aec37aff659fbe25aa416d3d8 --- /dev/null +++ b/checkpoints/000500/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d16c6607fd22ed0b16358fe1a965980e0f3d6559dc276656862d81943ddc2c5 +size 412659164 diff --git a/checkpoints/000500/training_state/rng_state.safetensors b/checkpoints/000500/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..6c9c21e647370409ea1e488e5786d480cd75d788 --- /dev/null +++ b/checkpoints/000500/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c17fb191d194f1c9b6303634639107f98e04dc8bcf1f31f28fa4334017055121 +size 15708 diff --git a/checkpoints/000500/training_state/scheduler_state.json b/checkpoints/000500/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..53425f04b861a6e9c20d98c7246e79420efec93e --- /dev/null +++ b/checkpoints/000500/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 2e-05 + ], + "last_epoch": 500, + "_step_count": 501, + "_is_initial": false, + "_get_lr_called_within_step": false, + "_last_lr": [ + 1.882772228311384e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/checkpoints/000500/training_state/training_step.json b/checkpoints/000500/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..8df625539d7e10ddfbd1e2ac0daf2fdd64ff5c4b --- /dev/null +++ b/checkpoints/000500/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 500 +} \ No newline at end of file diff --git a/checkpoints/001000/pretrained_model/config.json b/checkpoints/001000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..3bdae2d4dfa4b23cad255243a126b98d8f5d62c6 --- /dev/null +++ b/checkpoints/001000/pretrained_model/config.json @@ -0,0 +1,111 @@ +{ + "type": "smolvla", + "n_obs_steps": 1, + "input_features": { + "observation.images.image": { + "type": "VISUAL", + "shape": [ + 256, + 256, + 3 + ] + }, + "observation.images.wrist_image": { + "type": "VISUAL", + "shape": [ + 256, + 256, + 3 + ] + }, + "observation.state": { + "type": "STATE", + "shape": [ + 8 + ] + }, + "observation.states.ee_state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.states.joint_state": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "observation.states.gripper_state": { + "type": "STATE", + "shape": [ + 2 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "device": "cuda", + "use_amp": false, + "use_peft": false, + "push_to_hub": true, + "repo_id": "ardalon/sequential_t1_task48_fft", + "private": null, + "tags": null, + "license": null, + "pretrained_path": "/content/smolvla_base", + "chunk_size": 50, + "n_action_steps": 50, + "normalization_mapping": { + "VISUAL": "IDENTITY", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "max_state_dim": 32, + "max_action_dim": 32, + "resize_imgs_with_padding": [ + 512, + 512 + ], + "empty_cameras": 0, + "adapt_to_pi_aloha": false, + "use_delta_joint_actions_aloha": false, + "tokenizer_max_length": 48, + "num_steps": 10, + "use_cache": true, + "freeze_vision_encoder": true, + "train_expert_only": true, + "train_state_proj": true, + "optimizer_lr": 2e-05, + "optimizer_betas": [ + 0.9, + 0.95 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 0.01, + "optimizer_grad_clip_norm": 10.0, + "scheduler_warmup_steps": 1000, + "scheduler_decay_steps": 30000, + "scheduler_decay_lr": 2.5e-06, + "vlm_model_name": "HuggingFaceTB/SmolVLM2-500M-Video-Instruct", + "load_vlm_weights": true, + "add_image_special_tokens": false, + "attention_mode": "cross_attn", + "prefix_length": 0, + "pad_language_to": "max_length", + "num_expert_layers": 0, + "num_vlm_layers": 16, + "self_attn_every_n_layers": 2, + "expert_width_multiplier": 0.75, + "min_period": 0.004, + "max_period": 4.0, + "rtc_config": null, + "compile_model": false, + "compile_mode": "max-autotune" +} \ No newline at end of file diff --git a/checkpoints/001000/pretrained_model/model.safetensors b/checkpoints/001000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5b45b117a3ed2963c9486e115e378e8b2071242c --- /dev/null +++ b/checkpoints/001000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07a409f785cca4e0c224f67dfec6177017ac356a2523eb43eaaa7d644f53102b +size 906712520 diff --git a/checkpoints/001000/pretrained_model/policy_postprocessor.json b/checkpoints/001000/pretrained_model/policy_postprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..0ec8146a07f9b92bc5a3c02e2d6b7d49ba6d26e8 --- /dev/null +++ b/checkpoints/001000/pretrained_model/policy_postprocessor.json @@ -0,0 +1,32 @@ +{ + "name": "policy_postprocessor", + "steps": [ + { + "registry_name": "unnormalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "norm_map": { + "VISUAL": "IDENTITY", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + } + }, + "state_file": "policy_postprocessor_step_0_unnormalizer_processor.safetensors" + }, + { + "registry_name": "device_processor", + "config": { + "device": "cpu", + "float_dtype": null + } + } + ] +} \ No newline at end of file diff --git a/checkpoints/001000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors b/checkpoints/001000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..fb3b6bb870bf669cd89d6bc4600478c768d22459 --- /dev/null +++ b/checkpoints/001000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2abc5a6a77ada2e972696d0b4da9485e8ac2c2036629207a51cf587b2c434abb +size 5440 diff --git a/checkpoints/001000/pretrained_model/policy_preprocessor.json b/checkpoints/001000/pretrained_model/policy_preprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..7527b68cb0a2d7c7a1a0f858ac44bec3df5ac0bf --- /dev/null +++ b/checkpoints/001000/pretrained_model/policy_preprocessor.json @@ -0,0 +1,97 @@ +{ + "name": "policy_preprocessor", + "steps": [ + { + "registry_name": "rename_observations_processor", + "config": { + "rename_map": {} + } + }, + { + "registry_name": "to_batch_processor", + "config": {} + }, + { + "registry_name": "smolvla_new_line_processor", + "config": {} + }, + { + "registry_name": "tokenizer_processor", + "config": { + "max_length": 48, + "task_key": "task", + "padding_side": "right", + "padding": "max_length", + "truncation": true, + "tokenizer_name": "HuggingFaceTB/SmolVLM2-500M-Video-Instruct" + } + }, + { + "registry_name": "device_processor", + "config": { + "device": "cuda", + "float_dtype": null + } + }, + { + "registry_name": "normalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "observation.images.image": { + "type": "VISUAL", + "shape": [ + 256, + 256, + 3 + ] + }, + "observation.images.wrist_image": { + "type": "VISUAL", + "shape": [ + 256, + 256, + 3 + ] + }, + "observation.state": { + "type": "STATE", + "shape": [ + 8 + ] + }, + "observation.states.ee_state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.states.joint_state": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "observation.states.gripper_state": { + "type": "STATE", + "shape": [ + 2 + ] + }, + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "norm_map": { + "VISUAL": "IDENTITY", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + } + }, + "state_file": "policy_preprocessor_step_5_normalizer_processor.safetensors" + } + ] +} \ No newline at end of file diff --git a/checkpoints/001000/pretrained_model/policy_preprocessor_step_5_normalizer_processor.safetensors b/checkpoints/001000/pretrained_model/policy_preprocessor_step_5_normalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..fb3b6bb870bf669cd89d6bc4600478c768d22459 --- /dev/null +++ b/checkpoints/001000/pretrained_model/policy_preprocessor_step_5_normalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2abc5a6a77ada2e972696d0b4da9485e8ac2c2036629207a51cf587b2c434abb +size 5440 diff --git a/checkpoints/001000/pretrained_model/train_config.json b/checkpoints/001000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..06c696e54ad5d01142c45fe44d4d30291256fb50 --- /dev/null +++ b/checkpoints/001000/pretrained_model/train_config.json @@ -0,0 +1,392 @@ +{ + "dataset": { + "repo_id": "daniecraig/libero_90_no_noops_lerobot_v30", + "root": null, + "episodes": [ + 3092, + 3093, + 3094, + 3095, + 3096, + 3097, + 3098, + 3099, + 3100, + 3101, + 3102, + 3103, + 3104, + 3105, + 3106, + 3107, + 3108, + 3109, + 3110, + 3111, + 3112, + 3113, + 3114, + 3115, + 3116, + 3117, + 3118, + 3119, + 3120, + 3121, + 3122, + 3123, + 3124, + 3125, + 3126, + 3127, + 3128, + 3129, + 3130, + 3131, + 3132, + 3133, + 3134, + 3135, + 3136 + ], + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + }, + "affine": { + "weight": 1.0, + "type": "RandomAffine", + "kwargs": { + "degrees": [ + -5.0, + 5.0 + ], + "translate": [ + 0.05, + 0.05 + ] + } + } + } + }, + "revision": "main", + "use_imagenet_stats": true, + "video_backend": "torchcodec", + "streaming": false + }, + "env": { + "type": "libero", + "task": "libero_90", + "fps": 30, + "features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + }, + "pixels/agentview_image": { + "type": "VISUAL", + "shape": [ + 360, + 360, + 3 + ] + }, + "pixels/robot0_eye_in_hand_image": { + "type": "VISUAL", + "shape": [ + 360, + 360, + 3 + ] + }, + "robot_state/eef/pos": { + "type": "STATE", + "shape": [ + 3 + ] + }, + "robot_state/eef/quat": { + "type": "STATE", + "shape": [ + 4 + ] + }, + "robot_state/eef/mat": { + "type": "STATE", + "shape": [ + 3, + 3 + ] + }, + "robot_state/gripper/qpos": { + "type": "STATE", + "shape": [ + 2 + ] + }, + "robot_state/gripper/qvel": { + "type": "STATE", + "shape": [ + 2 + ] + }, + "robot_state/joints/pos": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "robot_state/joints/vel": { + "type": "STATE", + "shape": [ + 7 + ] + } + }, + "features_map": { + "action": "action", + "robot_state/eef/pos": "observation.state.eef_pos", + "robot_state/eef/quat": "observation.state.eef_quat", + "robot_state/eef/mat": "observation.state.eef_mat", + "robot_state/gripper/qpos": "observation.state.gripper_qpos", + "robot_state/gripper/qvel": "observation.state.gripper_qvel", + "robot_state/joints/pos": "observation.state.joint_pos", + "robot_state/joints/vel": "observation.state.joint_vel", + "pixels/agentview_image": "observation.images.image", + "pixels/robot0_eye_in_hand_image": "observation.images.image2" + }, + "max_parallel_tasks": 1, + "disable_env_checker": true, + "task_ids": [ + 48 + ], + "episode_length": null, + "obs_type": "pixels_agent_pos", + "render_mode": "rgb_array", + "camera_name": "agentview_image,robot0_eye_in_hand_image", + "init_states": true, + "camera_name_mapping": null, + "observation_height": 360, + "observation_width": 360, + "control_mode": "relative" + }, + "policy": { + "type": "smolvla", + "n_obs_steps": 1, + "input_features": { + "observation.images.image": { + "type": "VISUAL", + "shape": [ + 256, + 256, + 3 + ] + }, + "observation.images.wrist_image": { + "type": "VISUAL", + "shape": [ + 256, + 256, + 3 + ] + }, + "observation.state": { + "type": "STATE", + "shape": [ + 8 + ] + }, + "observation.states.ee_state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.states.joint_state": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "observation.states.gripper_state": { + "type": "STATE", + "shape": [ + 2 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "device": "cuda", + "use_amp": false, + "use_peft": false, + "push_to_hub": true, + "repo_id": "ardalon/sequential_t1_task48_fft", + "private": null, + "tags": null, + "license": null, + "pretrained_path": "/content/smolvla_base", + "chunk_size": 50, + "n_action_steps": 50, + "normalization_mapping": { + "VISUAL": "IDENTITY", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "max_state_dim": 32, + "max_action_dim": 32, + "resize_imgs_with_padding": [ + 512, + 512 + ], + "empty_cameras": 0, + "adapt_to_pi_aloha": false, + "use_delta_joint_actions_aloha": false, + "tokenizer_max_length": 48, + "num_steps": 10, + "use_cache": true, + "freeze_vision_encoder": true, + "train_expert_only": true, + "train_state_proj": true, + "optimizer_lr": 2e-05, + "optimizer_betas": [ + 0.9, + 0.95 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 0.01, + "optimizer_grad_clip_norm": 10.0, + "scheduler_warmup_steps": 1000, + "scheduler_decay_steps": 30000, + "scheduler_decay_lr": 2.5e-06, + "vlm_model_name": "HuggingFaceTB/SmolVLM2-500M-Video-Instruct", + "load_vlm_weights": true, + "add_image_special_tokens": false, + "attention_mode": "cross_attn", + "prefix_length": 0, + "pad_language_to": "max_length", + "num_expert_layers": 0, + "num_vlm_layers": 16, + "self_attn_every_n_layers": 2, + "expert_width_multiplier": 0.75, + "min_period": 0.004, + "max_period": 4.0, + "rtc_config": null, + "compile_model": false, + "compile_mode": "max-autotune" + }, + "output_dir": "outputs/train/sequential_t1_task48_fft", + "job_name": "sequential_t1_task48_fft", + "resume": false, + "seed": 42, + "cudnn_deterministic": false, + "num_workers": 4, + "batch_size": 16, + "steps": 3000, + "eval_freq": 0, + "log_freq": 1, + "tolerance_s": 0.0001, + "save_checkpoint": true, + "save_freq": 500, + "use_policy_training_preset": true, + "optimizer": { + "type": "adamw", + "lr": 2e-05, + "weight_decay": 0.01, + "grad_clip_norm": 10.0, + "betas": [ + 0.9, + 0.95 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "cosine_decay_with_warmup", + "num_warmup_steps": 1000, + "num_decay_steps": 30000, + "peak_lr": 2e-05, + "decay_lr": 2.5e-06 + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": true, + "project": "lerobot-smolvla", + "entity": null, + "notes": null, + "run_id": "ooipb6on", + "mode": null, + "add_tags": true + }, + "peft": null, + "use_rabc": false, + "rabc_progress_path": null, + "rabc_kappa": 0.01, + "rabc_epsilon": 1e-06, + "rabc_head_mode": "sparse", + "rename_map": {}, + "checkpoint_path": null +} \ No newline at end of file diff --git a/checkpoints/001000/training_state/optimizer_param_groups.json b/checkpoints/001000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..04f9953526b52fe5aaae91e67daea9bc33cab95b --- /dev/null +++ b/checkpoints/001000/training_state/optimizer_param_groups.json @@ -0,0 +1,521 @@ +[ + { + "lr": 1.5625e-05, + "betas": [ + 0.9, + 0.95 + ], + "eps": 1e-08, + "weight_decay": 0.01, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "initial_lr": 2e-05, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310, + 311, + 312, + 313, + 314, + 315, + 316, + 317, + 318, + 319, + 320, + 321, + 322, + 323, + 324, + 325, + 326, + 327, + 328, + 329, + 330, + 331, + 332, + 333, + 334, + 335, + 336, + 337, + 338, + 339, + 340, + 341, + 342, + 343, + 344, + 345, + 346, + 347, + 348, + 349, + 350, + 351, + 352, + 353, + 354, + 355, + 356, + 357, + 358, + 359, + 360, + 361, + 362, + 363, + 364, + 365, + 366, + 367, + 368, + 369, + 370, + 371, + 372, + 373, + 374, + 375, + 376, + 377, + 378, + 379, + 380, + 381, + 382, + 383, + 384, + 385, + 386, + 387, + 388, + 389, + 390, + 391, + 392, + 393, + 394, + 395, + 396, + 397, + 398, + 399, + 400, + 401, + 402, + 403, + 404, + 405, + 406, + 407, + 408, + 409, + 410, + 411, + 412, + 413, + 414, + 415, + 416, + 417, + 418, + 419, + 420, + 421, + 422, + 423, + 424, + 425, + 426, + 427, + 428, + 429, + 430, + 431, + 432, + 433, + 434, + 435, + 436, + 437, + 438, + 439, + 440, + 441, + 442, + 443, + 444, + 445, + 446, + 447, + 448, + 449, + 450, + 451, + 452, + 453, + 454, + 455, + 456, + 457, + 458, + 459, + 460, + 461, + 462, + 463, + 464, + 465, + 466, + 467, + 468, + 469, + 470, + 471, + 472, + 473, + 474, + 475, + 476, + 477, + 478, + 479, + 480, + 481, + 482, + 483, + 484, + 485, + 486, + 487, + 488, + 489, + 490, + 491, + 492, + 493, + 494, + 495, + 496, + 497, + 498, + 499 + ] + } +] \ No newline at end of file diff --git a/checkpoints/001000/training_state/optimizer_state.safetensors b/checkpoints/001000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3391361416a79253a0e2dfe9a4db95ae1adf35f2 --- /dev/null +++ b/checkpoints/001000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5d327599e74e6876918f0898b1687f9cddd60d8c6add611f7b55d06b73b6474 +size 412659164 diff --git a/checkpoints/001000/training_state/rng_state.safetensors b/checkpoints/001000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a86b1f2261ef7511c928b7cfce93b05bd31ac6d8 --- /dev/null +++ b/checkpoints/001000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b038f8968092299dde12dcb7abb70d0c81131cbe321136458b84c8acafad2bf +size 15708 diff --git a/checkpoints/001000/training_state/scheduler_state.json b/checkpoints/001000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..54d7fec4474a14b69423ab602ae3859e6602b5d2 --- /dev/null +++ b/checkpoints/001000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 2e-05 + ], + "last_epoch": 1000, + "_step_count": 1001, + "_is_initial": false, + "_get_lr_called_within_step": false, + "_last_lr": [ + 1.5625e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/checkpoints/001000/training_state/training_step.json b/checkpoints/001000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..d98c94f5b78238bf495ac68b9f9fb446cfac5c07 --- /dev/null +++ b/checkpoints/001000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 1000 +} \ No newline at end of file diff --git a/checkpoints/001500/pretrained_model/config.json b/checkpoints/001500/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..3bdae2d4dfa4b23cad255243a126b98d8f5d62c6 --- /dev/null +++ b/checkpoints/001500/pretrained_model/config.json @@ -0,0 +1,111 @@ +{ + "type": "smolvla", + "n_obs_steps": 1, + "input_features": { + "observation.images.image": { + "type": "VISUAL", + "shape": [ + 256, + 256, + 3 + ] + }, + "observation.images.wrist_image": { + "type": "VISUAL", + "shape": [ + 256, + 256, + 3 + ] + }, + "observation.state": { + "type": "STATE", + "shape": [ + 8 + ] + }, + "observation.states.ee_state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.states.joint_state": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "observation.states.gripper_state": { + "type": "STATE", + "shape": [ + 2 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "device": "cuda", + "use_amp": false, + "use_peft": false, + "push_to_hub": true, + "repo_id": "ardalon/sequential_t1_task48_fft", + "private": null, + "tags": null, + "license": null, + "pretrained_path": "/content/smolvla_base", + "chunk_size": 50, + "n_action_steps": 50, + "normalization_mapping": { + "VISUAL": "IDENTITY", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "max_state_dim": 32, + "max_action_dim": 32, + "resize_imgs_with_padding": [ + 512, + 512 + ], + "empty_cameras": 0, + "adapt_to_pi_aloha": false, + "use_delta_joint_actions_aloha": false, + "tokenizer_max_length": 48, + "num_steps": 10, + "use_cache": true, + "freeze_vision_encoder": true, + "train_expert_only": true, + "train_state_proj": true, + "optimizer_lr": 2e-05, + "optimizer_betas": [ + 0.9, + 0.95 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 0.01, + "optimizer_grad_clip_norm": 10.0, + "scheduler_warmup_steps": 1000, + "scheduler_decay_steps": 30000, + "scheduler_decay_lr": 2.5e-06, + "vlm_model_name": "HuggingFaceTB/SmolVLM2-500M-Video-Instruct", + "load_vlm_weights": true, + "add_image_special_tokens": false, + "attention_mode": "cross_attn", + "prefix_length": 0, + "pad_language_to": "max_length", + "num_expert_layers": 0, + "num_vlm_layers": 16, + "self_attn_every_n_layers": 2, + "expert_width_multiplier": 0.75, + "min_period": 0.004, + "max_period": 4.0, + "rtc_config": null, + "compile_model": false, + "compile_mode": "max-autotune" +} \ No newline at end of file diff --git a/checkpoints/001500/pretrained_model/model.safetensors b/checkpoints/001500/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..122d0c63d631cb38d111f008373ed1de26461ba2 --- /dev/null +++ b/checkpoints/001500/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e46cf45c79c6a51ab5f317337360c9da6d5129be51decc401abf3b5095216ee4 +size 906712520 diff --git a/checkpoints/001500/pretrained_model/policy_postprocessor.json b/checkpoints/001500/pretrained_model/policy_postprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..0ec8146a07f9b92bc5a3c02e2d6b7d49ba6d26e8 --- /dev/null +++ b/checkpoints/001500/pretrained_model/policy_postprocessor.json @@ -0,0 +1,32 @@ +{ + "name": "policy_postprocessor", + "steps": [ + { + "registry_name": "unnormalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "norm_map": { + "VISUAL": "IDENTITY", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + } + }, + "state_file": "policy_postprocessor_step_0_unnormalizer_processor.safetensors" + }, + { + "registry_name": "device_processor", + "config": { + "device": "cpu", + "float_dtype": null + } + } + ] +} \ No newline at end of file diff --git a/checkpoints/001500/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors b/checkpoints/001500/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..fb3b6bb870bf669cd89d6bc4600478c768d22459 --- /dev/null +++ b/checkpoints/001500/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2abc5a6a77ada2e972696d0b4da9485e8ac2c2036629207a51cf587b2c434abb +size 5440 diff --git a/checkpoints/001500/pretrained_model/policy_preprocessor.json b/checkpoints/001500/pretrained_model/policy_preprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..7527b68cb0a2d7c7a1a0f858ac44bec3df5ac0bf --- /dev/null +++ b/checkpoints/001500/pretrained_model/policy_preprocessor.json @@ -0,0 +1,97 @@ +{ + "name": "policy_preprocessor", + "steps": [ + { + "registry_name": "rename_observations_processor", + "config": { + "rename_map": {} + } + }, + { + "registry_name": "to_batch_processor", + "config": {} + }, + { + "registry_name": "smolvla_new_line_processor", + "config": {} + }, + { + "registry_name": "tokenizer_processor", + "config": { + "max_length": 48, + "task_key": "task", + "padding_side": "right", + "padding": "max_length", + "truncation": true, + "tokenizer_name": "HuggingFaceTB/SmolVLM2-500M-Video-Instruct" + } + }, + { + "registry_name": "device_processor", + "config": { + "device": "cuda", + "float_dtype": null + } + }, + { + "registry_name": "normalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "observation.images.image": { + "type": "VISUAL", + "shape": [ + 256, + 256, + 3 + ] + }, + "observation.images.wrist_image": { + "type": "VISUAL", + "shape": [ + 256, + 256, + 3 + ] + }, + "observation.state": { + "type": "STATE", + "shape": [ + 8 + ] + }, + "observation.states.ee_state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.states.joint_state": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "observation.states.gripper_state": { + "type": "STATE", + "shape": [ + 2 + ] + }, + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "norm_map": { + "VISUAL": "IDENTITY", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + } + }, + "state_file": "policy_preprocessor_step_5_normalizer_processor.safetensors" + } + ] +} \ No newline at end of file diff --git a/checkpoints/001500/pretrained_model/policy_preprocessor_step_5_normalizer_processor.safetensors b/checkpoints/001500/pretrained_model/policy_preprocessor_step_5_normalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..fb3b6bb870bf669cd89d6bc4600478c768d22459 --- /dev/null +++ b/checkpoints/001500/pretrained_model/policy_preprocessor_step_5_normalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2abc5a6a77ada2e972696d0b4da9485e8ac2c2036629207a51cf587b2c434abb +size 5440 diff --git a/checkpoints/001500/pretrained_model/train_config.json b/checkpoints/001500/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..06c696e54ad5d01142c45fe44d4d30291256fb50 --- /dev/null +++ b/checkpoints/001500/pretrained_model/train_config.json @@ -0,0 +1,392 @@ +{ + "dataset": { + "repo_id": "daniecraig/libero_90_no_noops_lerobot_v30", + "root": null, + "episodes": [ + 3092, + 3093, + 3094, + 3095, + 3096, + 3097, + 3098, + 3099, + 3100, + 3101, + 3102, + 3103, + 3104, + 3105, + 3106, + 3107, + 3108, + 3109, + 3110, + 3111, + 3112, + 3113, + 3114, + 3115, + 3116, + 3117, + 3118, + 3119, + 3120, + 3121, + 3122, + 3123, + 3124, + 3125, + 3126, + 3127, + 3128, + 3129, + 3130, + 3131, + 3132, + 3133, + 3134, + 3135, + 3136 + ], + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + }, + "affine": { + "weight": 1.0, + "type": "RandomAffine", + "kwargs": { + "degrees": [ + -5.0, + 5.0 + ], + "translate": [ + 0.05, + 0.05 + ] + } + } + } + }, + "revision": "main", + "use_imagenet_stats": true, + "video_backend": "torchcodec", + "streaming": false + }, + "env": { + "type": "libero", + "task": "libero_90", + "fps": 30, + "features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + }, + "pixels/agentview_image": { + "type": "VISUAL", + "shape": [ + 360, + 360, + 3 + ] + }, + "pixels/robot0_eye_in_hand_image": { + "type": "VISUAL", + "shape": [ + 360, + 360, + 3 + ] + }, + "robot_state/eef/pos": { + "type": "STATE", + "shape": [ + 3 + ] + }, + "robot_state/eef/quat": { + "type": "STATE", + "shape": [ + 4 + ] + }, + "robot_state/eef/mat": { + "type": "STATE", + "shape": [ + 3, + 3 + ] + }, + "robot_state/gripper/qpos": { + "type": "STATE", + "shape": [ + 2 + ] + }, + "robot_state/gripper/qvel": { + "type": "STATE", + "shape": [ + 2 + ] + }, + "robot_state/joints/pos": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "robot_state/joints/vel": { + "type": "STATE", + "shape": [ + 7 + ] + } + }, + "features_map": { + "action": "action", + "robot_state/eef/pos": "observation.state.eef_pos", + "robot_state/eef/quat": "observation.state.eef_quat", + "robot_state/eef/mat": "observation.state.eef_mat", + "robot_state/gripper/qpos": "observation.state.gripper_qpos", + "robot_state/gripper/qvel": "observation.state.gripper_qvel", + "robot_state/joints/pos": "observation.state.joint_pos", + "robot_state/joints/vel": "observation.state.joint_vel", + "pixels/agentview_image": "observation.images.image", + "pixels/robot0_eye_in_hand_image": "observation.images.image2" + }, + "max_parallel_tasks": 1, + "disable_env_checker": true, + "task_ids": [ + 48 + ], + "episode_length": null, + "obs_type": "pixels_agent_pos", + "render_mode": "rgb_array", + "camera_name": "agentview_image,robot0_eye_in_hand_image", + "init_states": true, + "camera_name_mapping": null, + "observation_height": 360, + "observation_width": 360, + "control_mode": "relative" + }, + "policy": { + "type": "smolvla", + "n_obs_steps": 1, + "input_features": { + "observation.images.image": { + "type": "VISUAL", + "shape": [ + 256, + 256, + 3 + ] + }, + "observation.images.wrist_image": { + "type": "VISUAL", + "shape": [ + 256, + 256, + 3 + ] + }, + "observation.state": { + "type": "STATE", + "shape": [ + 8 + ] + }, + "observation.states.ee_state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.states.joint_state": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "observation.states.gripper_state": { + "type": "STATE", + "shape": [ + 2 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "device": "cuda", + "use_amp": false, + "use_peft": false, + "push_to_hub": true, + "repo_id": "ardalon/sequential_t1_task48_fft", + "private": null, + "tags": null, + "license": null, + "pretrained_path": "/content/smolvla_base", + "chunk_size": 50, + "n_action_steps": 50, + "normalization_mapping": { + "VISUAL": "IDENTITY", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "max_state_dim": 32, + "max_action_dim": 32, + "resize_imgs_with_padding": [ + 512, + 512 + ], + "empty_cameras": 0, + "adapt_to_pi_aloha": false, + "use_delta_joint_actions_aloha": false, + "tokenizer_max_length": 48, + "num_steps": 10, + "use_cache": true, + "freeze_vision_encoder": true, + "train_expert_only": true, + "train_state_proj": true, + "optimizer_lr": 2e-05, + "optimizer_betas": [ + 0.9, + 0.95 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 0.01, + "optimizer_grad_clip_norm": 10.0, + "scheduler_warmup_steps": 1000, + "scheduler_decay_steps": 30000, + "scheduler_decay_lr": 2.5e-06, + "vlm_model_name": "HuggingFaceTB/SmolVLM2-500M-Video-Instruct", + "load_vlm_weights": true, + "add_image_special_tokens": false, + "attention_mode": "cross_attn", + "prefix_length": 0, + "pad_language_to": "max_length", + "num_expert_layers": 0, + "num_vlm_layers": 16, + "self_attn_every_n_layers": 2, + "expert_width_multiplier": 0.75, + "min_period": 0.004, + "max_period": 4.0, + "rtc_config": null, + "compile_model": false, + "compile_mode": "max-autotune" + }, + "output_dir": "outputs/train/sequential_t1_task48_fft", + "job_name": "sequential_t1_task48_fft", + "resume": false, + "seed": 42, + "cudnn_deterministic": false, + "num_workers": 4, + "batch_size": 16, + "steps": 3000, + "eval_freq": 0, + "log_freq": 1, + "tolerance_s": 0.0001, + "save_checkpoint": true, + "save_freq": 500, + "use_policy_training_preset": true, + "optimizer": { + "type": "adamw", + "lr": 2e-05, + "weight_decay": 0.01, + "grad_clip_norm": 10.0, + "betas": [ + 0.9, + 0.95 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "cosine_decay_with_warmup", + "num_warmup_steps": 1000, + "num_decay_steps": 30000, + "peak_lr": 2e-05, + "decay_lr": 2.5e-06 + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": true, + "project": "lerobot-smolvla", + "entity": null, + "notes": null, + "run_id": "ooipb6on", + "mode": null, + "add_tags": true + }, + "peft": null, + "use_rabc": false, + "rabc_progress_path": null, + "rabc_kappa": 0.01, + "rabc_epsilon": 1e-06, + "rabc_head_mode": "sparse", + "rename_map": {}, + "checkpoint_path": null +} \ No newline at end of file diff --git a/checkpoints/001500/training_state/optimizer_param_groups.json b/checkpoints/001500/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..c35b4c3a983b93541d9b27d44b230ee725ffb3e8 --- /dev/null +++ b/checkpoints/001500/training_state/optimizer_param_groups.json @@ -0,0 +1,521 @@ +[ + { + "lr": 1.125e-05, + "betas": [ + 0.9, + 0.95 + ], + "eps": 1e-08, + "weight_decay": 0.01, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "initial_lr": 2e-05, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310, + 311, + 312, + 313, + 314, + 315, + 316, + 317, + 318, + 319, + 320, + 321, + 322, + 323, + 324, + 325, + 326, + 327, + 328, + 329, + 330, + 331, + 332, + 333, + 334, + 335, + 336, + 337, + 338, + 339, + 340, + 341, + 342, + 343, + 344, + 345, + 346, + 347, + 348, + 349, + 350, + 351, + 352, + 353, + 354, + 355, + 356, + 357, + 358, + 359, + 360, + 361, + 362, + 363, + 364, + 365, + 366, + 367, + 368, + 369, + 370, + 371, + 372, + 373, + 374, + 375, + 376, + 377, + 378, + 379, + 380, + 381, + 382, + 383, + 384, + 385, + 386, + 387, + 388, + 389, + 390, + 391, + 392, + 393, + 394, + 395, + 396, + 397, + 398, + 399, + 400, + 401, + 402, + 403, + 404, + 405, + 406, + 407, + 408, + 409, + 410, + 411, + 412, + 413, + 414, + 415, + 416, + 417, + 418, + 419, + 420, + 421, + 422, + 423, + 424, + 425, + 426, + 427, + 428, + 429, + 430, + 431, + 432, + 433, + 434, + 435, + 436, + 437, + 438, + 439, + 440, + 441, + 442, + 443, + 444, + 445, + 446, + 447, + 448, + 449, + 450, + 451, + 452, + 453, + 454, + 455, + 456, + 457, + 458, + 459, + 460, + 461, + 462, + 463, + 464, + 465, + 466, + 467, + 468, + 469, + 470, + 471, + 472, + 473, + 474, + 475, + 476, + 477, + 478, + 479, + 480, + 481, + 482, + 483, + 484, + 485, + 486, + 487, + 488, + 489, + 490, + 491, + 492, + 493, + 494, + 495, + 496, + 497, + 498, + 499 + ] + } +] \ No newline at end of file diff --git a/checkpoints/001500/training_state/optimizer_state.safetensors b/checkpoints/001500/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f000d57f2c2187cf0e9fe9fca38ec4a3a5be0f33 --- /dev/null +++ b/checkpoints/001500/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d48a841571468d9d86757a11db4292eb5f5e293e788a6603df7488ca8d92b637 +size 412659164 diff --git a/checkpoints/001500/training_state/rng_state.safetensors b/checkpoints/001500/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a447459d21c683d63e748b4b0b15cb1a98d5b272 --- /dev/null +++ b/checkpoints/001500/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8be410a5c94dffdd8e67a609da364680c686f02f64ac6b41edcd106aa491a83 +size 15708 diff --git a/checkpoints/001500/training_state/scheduler_state.json b/checkpoints/001500/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..d5c891d0d6bf19fc0783d3df2afb10cee4e2e51c --- /dev/null +++ b/checkpoints/001500/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 2e-05 + ], + "last_epoch": 1500, + "_step_count": 1501, + "_is_initial": false, + "_get_lr_called_within_step": false, + "_last_lr": [ + 1.125e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/checkpoints/001500/training_state/training_step.json b/checkpoints/001500/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..332c190c78659d9403699dab54afdd72e41d21b8 --- /dev/null +++ b/checkpoints/001500/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 1500 +} \ No newline at end of file diff --git a/checkpoints/002000/pretrained_model/config.json b/checkpoints/002000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..3bdae2d4dfa4b23cad255243a126b98d8f5d62c6 --- /dev/null +++ b/checkpoints/002000/pretrained_model/config.json @@ -0,0 +1,111 @@ +{ + "type": "smolvla", + "n_obs_steps": 1, + "input_features": { + "observation.images.image": { + "type": "VISUAL", + "shape": [ + 256, + 256, + 3 + ] + }, + "observation.images.wrist_image": { + "type": "VISUAL", + "shape": [ + 256, + 256, + 3 + ] + }, + "observation.state": { + "type": "STATE", + "shape": [ + 8 + ] + }, + "observation.states.ee_state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.states.joint_state": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "observation.states.gripper_state": { + "type": "STATE", + "shape": [ + 2 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "device": "cuda", + "use_amp": false, + "use_peft": false, + "push_to_hub": true, + "repo_id": "ardalon/sequential_t1_task48_fft", + "private": null, + "tags": null, + "license": null, + "pretrained_path": "/content/smolvla_base", + "chunk_size": 50, + "n_action_steps": 50, + "normalization_mapping": { + "VISUAL": "IDENTITY", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "max_state_dim": 32, + "max_action_dim": 32, + "resize_imgs_with_padding": [ + 512, + 512 + ], + "empty_cameras": 0, + "adapt_to_pi_aloha": false, + "use_delta_joint_actions_aloha": false, + "tokenizer_max_length": 48, + "num_steps": 10, + "use_cache": true, + "freeze_vision_encoder": true, + "train_expert_only": true, + "train_state_proj": true, + "optimizer_lr": 2e-05, + "optimizer_betas": [ + 0.9, + 0.95 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 0.01, + "optimizer_grad_clip_norm": 10.0, + "scheduler_warmup_steps": 1000, + "scheduler_decay_steps": 30000, + "scheduler_decay_lr": 2.5e-06, + "vlm_model_name": "HuggingFaceTB/SmolVLM2-500M-Video-Instruct", + "load_vlm_weights": true, + "add_image_special_tokens": false, + "attention_mode": "cross_attn", + "prefix_length": 0, + "pad_language_to": "max_length", + "num_expert_layers": 0, + "num_vlm_layers": 16, + "self_attn_every_n_layers": 2, + "expert_width_multiplier": 0.75, + "min_period": 0.004, + "max_period": 4.0, + "rtc_config": null, + "compile_model": false, + "compile_mode": "max-autotune" +} \ No newline at end of file diff --git a/checkpoints/002000/pretrained_model/model.safetensors b/checkpoints/002000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..03042508ece466f68e3f479313d2c3424dc9c894 --- /dev/null +++ b/checkpoints/002000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:171400216513e269701326300b60c2f615809872e2637435ffff06bd15163cc0 +size 906712520 diff --git a/checkpoints/002000/pretrained_model/policy_postprocessor.json b/checkpoints/002000/pretrained_model/policy_postprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..0ec8146a07f9b92bc5a3c02e2d6b7d49ba6d26e8 --- /dev/null +++ b/checkpoints/002000/pretrained_model/policy_postprocessor.json @@ -0,0 +1,32 @@ +{ + "name": "policy_postprocessor", + "steps": [ + { + "registry_name": "unnormalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "norm_map": { + "VISUAL": "IDENTITY", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + } + }, + "state_file": "policy_postprocessor_step_0_unnormalizer_processor.safetensors" + }, + { + "registry_name": "device_processor", + "config": { + "device": "cpu", + "float_dtype": null + } + } + ] +} \ No newline at end of file diff --git a/checkpoints/002000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors b/checkpoints/002000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..fb3b6bb870bf669cd89d6bc4600478c768d22459 --- /dev/null +++ b/checkpoints/002000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2abc5a6a77ada2e972696d0b4da9485e8ac2c2036629207a51cf587b2c434abb +size 5440 diff --git a/checkpoints/002000/pretrained_model/policy_preprocessor.json b/checkpoints/002000/pretrained_model/policy_preprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..7527b68cb0a2d7c7a1a0f858ac44bec3df5ac0bf --- /dev/null +++ b/checkpoints/002000/pretrained_model/policy_preprocessor.json @@ -0,0 +1,97 @@ +{ + "name": "policy_preprocessor", + "steps": [ + { + "registry_name": "rename_observations_processor", + "config": { + "rename_map": {} + } + }, + { + "registry_name": "to_batch_processor", + "config": {} + }, + { + "registry_name": "smolvla_new_line_processor", + "config": {} + }, + { + "registry_name": "tokenizer_processor", + "config": { + "max_length": 48, + "task_key": "task", + "padding_side": "right", + "padding": "max_length", + "truncation": true, + "tokenizer_name": "HuggingFaceTB/SmolVLM2-500M-Video-Instruct" + } + }, + { + "registry_name": "device_processor", + "config": { + "device": "cuda", + "float_dtype": null + } + }, + { + "registry_name": "normalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "observation.images.image": { + "type": "VISUAL", + "shape": [ + 256, + 256, + 3 + ] + }, + "observation.images.wrist_image": { + "type": "VISUAL", + "shape": [ + 256, + 256, + 3 + ] + }, + "observation.state": { + "type": "STATE", + "shape": [ + 8 + ] + }, + "observation.states.ee_state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.states.joint_state": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "observation.states.gripper_state": { + "type": "STATE", + "shape": [ + 2 + ] + }, + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "norm_map": { + "VISUAL": "IDENTITY", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + } + }, + "state_file": "policy_preprocessor_step_5_normalizer_processor.safetensors" + } + ] +} \ No newline at end of file diff --git a/checkpoints/002000/pretrained_model/policy_preprocessor_step_5_normalizer_processor.safetensors b/checkpoints/002000/pretrained_model/policy_preprocessor_step_5_normalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..fb3b6bb870bf669cd89d6bc4600478c768d22459 --- /dev/null +++ b/checkpoints/002000/pretrained_model/policy_preprocessor_step_5_normalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2abc5a6a77ada2e972696d0b4da9485e8ac2c2036629207a51cf587b2c434abb +size 5440 diff --git a/checkpoints/002000/pretrained_model/train_config.json b/checkpoints/002000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..06c696e54ad5d01142c45fe44d4d30291256fb50 --- /dev/null +++ b/checkpoints/002000/pretrained_model/train_config.json @@ -0,0 +1,392 @@ +{ + "dataset": { + "repo_id": "daniecraig/libero_90_no_noops_lerobot_v30", + "root": null, + "episodes": [ + 3092, + 3093, + 3094, + 3095, + 3096, + 3097, + 3098, + 3099, + 3100, + 3101, + 3102, + 3103, + 3104, + 3105, + 3106, + 3107, + 3108, + 3109, + 3110, + 3111, + 3112, + 3113, + 3114, + 3115, + 3116, + 3117, + 3118, + 3119, + 3120, + 3121, + 3122, + 3123, + 3124, + 3125, + 3126, + 3127, + 3128, + 3129, + 3130, + 3131, + 3132, + 3133, + 3134, + 3135, + 3136 + ], + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + }, + "affine": { + "weight": 1.0, + "type": "RandomAffine", + "kwargs": { + "degrees": [ + -5.0, + 5.0 + ], + "translate": [ + 0.05, + 0.05 + ] + } + } + } + }, + "revision": "main", + "use_imagenet_stats": true, + "video_backend": "torchcodec", + "streaming": false + }, + "env": { + "type": "libero", + "task": "libero_90", + "fps": 30, + "features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + }, + "pixels/agentview_image": { + "type": "VISUAL", + "shape": [ + 360, + 360, + 3 + ] + }, + "pixels/robot0_eye_in_hand_image": { + "type": "VISUAL", + "shape": [ + 360, + 360, + 3 + ] + }, + "robot_state/eef/pos": { + "type": "STATE", + "shape": [ + 3 + ] + }, + "robot_state/eef/quat": { + "type": "STATE", + "shape": [ + 4 + ] + }, + "robot_state/eef/mat": { + "type": "STATE", + "shape": [ + 3, + 3 + ] + }, + "robot_state/gripper/qpos": { + "type": "STATE", + "shape": [ + 2 + ] + }, + "robot_state/gripper/qvel": { + "type": "STATE", + "shape": [ + 2 + ] + }, + "robot_state/joints/pos": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "robot_state/joints/vel": { + "type": "STATE", + "shape": [ + 7 + ] + } + }, + "features_map": { + "action": "action", + "robot_state/eef/pos": "observation.state.eef_pos", + "robot_state/eef/quat": "observation.state.eef_quat", + "robot_state/eef/mat": "observation.state.eef_mat", + "robot_state/gripper/qpos": "observation.state.gripper_qpos", + "robot_state/gripper/qvel": "observation.state.gripper_qvel", + "robot_state/joints/pos": "observation.state.joint_pos", + "robot_state/joints/vel": "observation.state.joint_vel", + "pixels/agentview_image": "observation.images.image", + "pixels/robot0_eye_in_hand_image": "observation.images.image2" + }, + "max_parallel_tasks": 1, + "disable_env_checker": true, + "task_ids": [ + 48 + ], + "episode_length": null, + "obs_type": "pixels_agent_pos", + "render_mode": "rgb_array", + "camera_name": "agentview_image,robot0_eye_in_hand_image", + "init_states": true, + "camera_name_mapping": null, + "observation_height": 360, + "observation_width": 360, + "control_mode": "relative" + }, + "policy": { + "type": "smolvla", + "n_obs_steps": 1, + "input_features": { + "observation.images.image": { + "type": "VISUAL", + "shape": [ + 256, + 256, + 3 + ] + }, + "observation.images.wrist_image": { + "type": "VISUAL", + "shape": [ + 256, + 256, + 3 + ] + }, + "observation.state": { + "type": "STATE", + "shape": [ + 8 + ] + }, + "observation.states.ee_state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.states.joint_state": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "observation.states.gripper_state": { + "type": "STATE", + "shape": [ + 2 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "device": "cuda", + "use_amp": false, + "use_peft": false, + "push_to_hub": true, + "repo_id": "ardalon/sequential_t1_task48_fft", + "private": null, + "tags": null, + "license": null, + "pretrained_path": "/content/smolvla_base", + "chunk_size": 50, + "n_action_steps": 50, + "normalization_mapping": { + "VISUAL": "IDENTITY", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "max_state_dim": 32, + "max_action_dim": 32, + "resize_imgs_with_padding": [ + 512, + 512 + ], + "empty_cameras": 0, + "adapt_to_pi_aloha": false, + "use_delta_joint_actions_aloha": false, + "tokenizer_max_length": 48, + "num_steps": 10, + "use_cache": true, + "freeze_vision_encoder": true, + "train_expert_only": true, + "train_state_proj": true, + "optimizer_lr": 2e-05, + "optimizer_betas": [ + 0.9, + 0.95 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 0.01, + "optimizer_grad_clip_norm": 10.0, + "scheduler_warmup_steps": 1000, + "scheduler_decay_steps": 30000, + "scheduler_decay_lr": 2.5e-06, + "vlm_model_name": "HuggingFaceTB/SmolVLM2-500M-Video-Instruct", + "load_vlm_weights": true, + "add_image_special_tokens": false, + "attention_mode": "cross_attn", + "prefix_length": 0, + "pad_language_to": "max_length", + "num_expert_layers": 0, + "num_vlm_layers": 16, + "self_attn_every_n_layers": 2, + "expert_width_multiplier": 0.75, + "min_period": 0.004, + "max_period": 4.0, + "rtc_config": null, + "compile_model": false, + "compile_mode": "max-autotune" + }, + "output_dir": "outputs/train/sequential_t1_task48_fft", + "job_name": "sequential_t1_task48_fft", + "resume": false, + "seed": 42, + "cudnn_deterministic": false, + "num_workers": 4, + "batch_size": 16, + "steps": 3000, + "eval_freq": 0, + "log_freq": 1, + "tolerance_s": 0.0001, + "save_checkpoint": true, + "save_freq": 500, + "use_policy_training_preset": true, + "optimizer": { + "type": "adamw", + "lr": 2e-05, + "weight_decay": 0.01, + "grad_clip_norm": 10.0, + "betas": [ + 0.9, + 0.95 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "cosine_decay_with_warmup", + "num_warmup_steps": 1000, + "num_decay_steps": 30000, + "peak_lr": 2e-05, + "decay_lr": 2.5e-06 + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": true, + "project": "lerobot-smolvla", + "entity": null, + "notes": null, + "run_id": "ooipb6on", + "mode": null, + "add_tags": true + }, + "peft": null, + "use_rabc": false, + "rabc_progress_path": null, + "rabc_kappa": 0.01, + "rabc_epsilon": 1e-06, + "rabc_head_mode": "sparse", + "rename_map": {}, + "checkpoint_path": null +} \ No newline at end of file diff --git a/checkpoints/002000/training_state/optimizer_param_groups.json b/checkpoints/002000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..2c47e46147541f06ec2b940d4ab4a8e8a3d63c2f --- /dev/null +++ b/checkpoints/002000/training_state/optimizer_param_groups.json @@ -0,0 +1,521 @@ +[ + { + "lr": 6.875000000000003e-06, + "betas": [ + 0.9, + 0.95 + ], + "eps": 1e-08, + "weight_decay": 0.01, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "initial_lr": 2e-05, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310, + 311, + 312, + 313, + 314, + 315, + 316, + 317, + 318, + 319, + 320, + 321, + 322, + 323, + 324, + 325, + 326, + 327, + 328, + 329, + 330, + 331, + 332, + 333, + 334, + 335, + 336, + 337, + 338, + 339, + 340, + 341, + 342, + 343, + 344, + 345, + 346, + 347, + 348, + 349, + 350, + 351, + 352, + 353, + 354, + 355, + 356, + 357, + 358, + 359, + 360, + 361, + 362, + 363, + 364, + 365, + 366, + 367, + 368, + 369, + 370, + 371, + 372, + 373, + 374, + 375, + 376, + 377, + 378, + 379, + 380, + 381, + 382, + 383, + 384, + 385, + 386, + 387, + 388, + 389, + 390, + 391, + 392, + 393, + 394, + 395, + 396, + 397, + 398, + 399, + 400, + 401, + 402, + 403, + 404, + 405, + 406, + 407, + 408, + 409, + 410, + 411, + 412, + 413, + 414, + 415, + 416, + 417, + 418, + 419, + 420, + 421, + 422, + 423, + 424, + 425, + 426, + 427, + 428, + 429, + 430, + 431, + 432, + 433, + 434, + 435, + 436, + 437, + 438, + 439, + 440, + 441, + 442, + 443, + 444, + 445, + 446, + 447, + 448, + 449, + 450, + 451, + 452, + 453, + 454, + 455, + 456, + 457, + 458, + 459, + 460, + 461, + 462, + 463, + 464, + 465, + 466, + 467, + 468, + 469, + 470, + 471, + 472, + 473, + 474, + 475, + 476, + 477, + 478, + 479, + 480, + 481, + 482, + 483, + 484, + 485, + 486, + 487, + 488, + 489, + 490, + 491, + 492, + 493, + 494, + 495, + 496, + 497, + 498, + 499 + ] + } +] \ No newline at end of file diff --git a/checkpoints/002000/training_state/optimizer_state.safetensors b/checkpoints/002000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7c208b0ece6c4d2dc16767d723ea5d0672b3b364 --- /dev/null +++ b/checkpoints/002000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b474899aabf79a4bba4e81fc6fa4674989864072bdcba977238012cc4779d847 +size 412659164 diff --git a/checkpoints/002000/training_state/rng_state.safetensors b/checkpoints/002000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b8c10b9192bb3de3959486a698226eff7b2da017 --- /dev/null +++ b/checkpoints/002000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac167752f5d5ce7c431d2d73f7b8cc4535a1014b68238184a8f4964ba30fc228 +size 15708 diff --git a/checkpoints/002000/training_state/scheduler_state.json b/checkpoints/002000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..24e195101ca89445e432632c9990a62c8b697a37 --- /dev/null +++ b/checkpoints/002000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 2e-05 + ], + "last_epoch": 2000, + "_step_count": 2001, + "_is_initial": false, + "_get_lr_called_within_step": false, + "_last_lr": [ + 6.875000000000003e-06 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/checkpoints/002000/training_state/training_step.json b/checkpoints/002000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..39090bbb986edb821e1602990d19357dcdb5d2ae --- /dev/null +++ b/checkpoints/002000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 2000 +} \ No newline at end of file diff --git a/checkpoints/002500/pretrained_model/config.json b/checkpoints/002500/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..3bdae2d4dfa4b23cad255243a126b98d8f5d62c6 --- /dev/null +++ b/checkpoints/002500/pretrained_model/config.json @@ -0,0 +1,111 @@ +{ + "type": "smolvla", + "n_obs_steps": 1, + "input_features": { + "observation.images.image": { + "type": "VISUAL", + "shape": [ + 256, + 256, + 3 + ] + }, + "observation.images.wrist_image": { + "type": "VISUAL", + "shape": [ + 256, + 256, + 3 + ] + }, + "observation.state": { + "type": "STATE", + "shape": [ + 8 + ] + }, + "observation.states.ee_state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.states.joint_state": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "observation.states.gripper_state": { + "type": "STATE", + "shape": [ + 2 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "device": "cuda", + "use_amp": false, + "use_peft": false, + "push_to_hub": true, + "repo_id": "ardalon/sequential_t1_task48_fft", + "private": null, + "tags": null, + "license": null, + "pretrained_path": "/content/smolvla_base", + "chunk_size": 50, + "n_action_steps": 50, + "normalization_mapping": { + "VISUAL": "IDENTITY", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "max_state_dim": 32, + "max_action_dim": 32, + "resize_imgs_with_padding": [ + 512, + 512 + ], + "empty_cameras": 0, + "adapt_to_pi_aloha": false, + "use_delta_joint_actions_aloha": false, + "tokenizer_max_length": 48, + "num_steps": 10, + "use_cache": true, + "freeze_vision_encoder": true, + "train_expert_only": true, + "train_state_proj": true, + "optimizer_lr": 2e-05, + "optimizer_betas": [ + 0.9, + 0.95 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 0.01, + "optimizer_grad_clip_norm": 10.0, + "scheduler_warmup_steps": 1000, + "scheduler_decay_steps": 30000, + "scheduler_decay_lr": 2.5e-06, + "vlm_model_name": "HuggingFaceTB/SmolVLM2-500M-Video-Instruct", + "load_vlm_weights": true, + "add_image_special_tokens": false, + "attention_mode": "cross_attn", + "prefix_length": 0, + "pad_language_to": "max_length", + "num_expert_layers": 0, + "num_vlm_layers": 16, + "self_attn_every_n_layers": 2, + "expert_width_multiplier": 0.75, + "min_period": 0.004, + "max_period": 4.0, + "rtc_config": null, + "compile_model": false, + "compile_mode": "max-autotune" +} \ No newline at end of file diff --git a/checkpoints/002500/pretrained_model/model.safetensors b/checkpoints/002500/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1e0229d319bfd6b1a882a767178458f8e5aaccee --- /dev/null +++ b/checkpoints/002500/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e55eb1fcac756616f2f3ded7728ed4d7c40ac7512eff37aec4508c23c7a718d +size 906712520 diff --git a/checkpoints/002500/pretrained_model/policy_postprocessor.json b/checkpoints/002500/pretrained_model/policy_postprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..0ec8146a07f9b92bc5a3c02e2d6b7d49ba6d26e8 --- /dev/null +++ b/checkpoints/002500/pretrained_model/policy_postprocessor.json @@ -0,0 +1,32 @@ +{ + "name": "policy_postprocessor", + "steps": [ + { + "registry_name": "unnormalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "norm_map": { + "VISUAL": "IDENTITY", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + } + }, + "state_file": "policy_postprocessor_step_0_unnormalizer_processor.safetensors" + }, + { + "registry_name": "device_processor", + "config": { + "device": "cpu", + "float_dtype": null + } + } + ] +} \ No newline at end of file diff --git a/checkpoints/002500/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors b/checkpoints/002500/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..fb3b6bb870bf669cd89d6bc4600478c768d22459 --- /dev/null +++ b/checkpoints/002500/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2abc5a6a77ada2e972696d0b4da9485e8ac2c2036629207a51cf587b2c434abb +size 5440 diff --git a/checkpoints/002500/pretrained_model/policy_preprocessor.json b/checkpoints/002500/pretrained_model/policy_preprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..7527b68cb0a2d7c7a1a0f858ac44bec3df5ac0bf --- /dev/null +++ b/checkpoints/002500/pretrained_model/policy_preprocessor.json @@ -0,0 +1,97 @@ +{ + "name": "policy_preprocessor", + "steps": [ + { + "registry_name": "rename_observations_processor", + "config": { + "rename_map": {} + } + }, + { + "registry_name": "to_batch_processor", + "config": {} + }, + { + "registry_name": "smolvla_new_line_processor", + "config": {} + }, + { + "registry_name": "tokenizer_processor", + "config": { + "max_length": 48, + "task_key": "task", + "padding_side": "right", + "padding": "max_length", + "truncation": true, + "tokenizer_name": "HuggingFaceTB/SmolVLM2-500M-Video-Instruct" + } + }, + { + "registry_name": "device_processor", + "config": { + "device": "cuda", + "float_dtype": null + } + }, + { + "registry_name": "normalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "observation.images.image": { + "type": "VISUAL", + "shape": [ + 256, + 256, + 3 + ] + }, + "observation.images.wrist_image": { + "type": "VISUAL", + "shape": [ + 256, + 256, + 3 + ] + }, + "observation.state": { + "type": "STATE", + "shape": [ + 8 + ] + }, + "observation.states.ee_state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.states.joint_state": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "observation.states.gripper_state": { + "type": "STATE", + "shape": [ + 2 + ] + }, + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "norm_map": { + "VISUAL": "IDENTITY", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + } + }, + "state_file": "policy_preprocessor_step_5_normalizer_processor.safetensors" + } + ] +} \ No newline at end of file diff --git a/checkpoints/002500/pretrained_model/policy_preprocessor_step_5_normalizer_processor.safetensors b/checkpoints/002500/pretrained_model/policy_preprocessor_step_5_normalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..fb3b6bb870bf669cd89d6bc4600478c768d22459 --- /dev/null +++ b/checkpoints/002500/pretrained_model/policy_preprocessor_step_5_normalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2abc5a6a77ada2e972696d0b4da9485e8ac2c2036629207a51cf587b2c434abb +size 5440 diff --git a/checkpoints/002500/pretrained_model/train_config.json b/checkpoints/002500/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..06c696e54ad5d01142c45fe44d4d30291256fb50 --- /dev/null +++ b/checkpoints/002500/pretrained_model/train_config.json @@ -0,0 +1,392 @@ +{ + "dataset": { + "repo_id": "daniecraig/libero_90_no_noops_lerobot_v30", + "root": null, + "episodes": [ + 3092, + 3093, + 3094, + 3095, + 3096, + 3097, + 3098, + 3099, + 3100, + 3101, + 3102, + 3103, + 3104, + 3105, + 3106, + 3107, + 3108, + 3109, + 3110, + 3111, + 3112, + 3113, + 3114, + 3115, + 3116, + 3117, + 3118, + 3119, + 3120, + 3121, + 3122, + 3123, + 3124, + 3125, + 3126, + 3127, + 3128, + 3129, + 3130, + 3131, + 3132, + 3133, + 3134, + 3135, + 3136 + ], + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + }, + "affine": { + "weight": 1.0, + "type": "RandomAffine", + "kwargs": { + "degrees": [ + -5.0, + 5.0 + ], + "translate": [ + 0.05, + 0.05 + ] + } + } + } + }, + "revision": "main", + "use_imagenet_stats": true, + "video_backend": "torchcodec", + "streaming": false + }, + "env": { + "type": "libero", + "task": "libero_90", + "fps": 30, + "features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + }, + "pixels/agentview_image": { + "type": "VISUAL", + "shape": [ + 360, + 360, + 3 + ] + }, + "pixels/robot0_eye_in_hand_image": { + "type": "VISUAL", + "shape": [ + 360, + 360, + 3 + ] + }, + "robot_state/eef/pos": { + "type": "STATE", + "shape": [ + 3 + ] + }, + "robot_state/eef/quat": { + "type": "STATE", + "shape": [ + 4 + ] + }, + "robot_state/eef/mat": { + "type": "STATE", + "shape": [ + 3, + 3 + ] + }, + "robot_state/gripper/qpos": { + "type": "STATE", + "shape": [ + 2 + ] + }, + "robot_state/gripper/qvel": { + "type": "STATE", + "shape": [ + 2 + ] + }, + "robot_state/joints/pos": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "robot_state/joints/vel": { + "type": "STATE", + "shape": [ + 7 + ] + } + }, + "features_map": { + "action": "action", + "robot_state/eef/pos": "observation.state.eef_pos", + "robot_state/eef/quat": "observation.state.eef_quat", + "robot_state/eef/mat": "observation.state.eef_mat", + "robot_state/gripper/qpos": "observation.state.gripper_qpos", + "robot_state/gripper/qvel": "observation.state.gripper_qvel", + "robot_state/joints/pos": "observation.state.joint_pos", + "robot_state/joints/vel": "observation.state.joint_vel", + "pixels/agentview_image": "observation.images.image", + "pixels/robot0_eye_in_hand_image": "observation.images.image2" + }, + "max_parallel_tasks": 1, + "disable_env_checker": true, + "task_ids": [ + 48 + ], + "episode_length": null, + "obs_type": "pixels_agent_pos", + "render_mode": "rgb_array", + "camera_name": "agentview_image,robot0_eye_in_hand_image", + "init_states": true, + "camera_name_mapping": null, + "observation_height": 360, + "observation_width": 360, + "control_mode": "relative" + }, + "policy": { + "type": "smolvla", + "n_obs_steps": 1, + "input_features": { + "observation.images.image": { + "type": "VISUAL", + "shape": [ + 256, + 256, + 3 + ] + }, + "observation.images.wrist_image": { + "type": "VISUAL", + "shape": [ + 256, + 256, + 3 + ] + }, + "observation.state": { + "type": "STATE", + "shape": [ + 8 + ] + }, + "observation.states.ee_state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.states.joint_state": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "observation.states.gripper_state": { + "type": "STATE", + "shape": [ + 2 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "device": "cuda", + "use_amp": false, + "use_peft": false, + "push_to_hub": true, + "repo_id": "ardalon/sequential_t1_task48_fft", + "private": null, + "tags": null, + "license": null, + "pretrained_path": "/content/smolvla_base", + "chunk_size": 50, + "n_action_steps": 50, + "normalization_mapping": { + "VISUAL": "IDENTITY", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "max_state_dim": 32, + "max_action_dim": 32, + "resize_imgs_with_padding": [ + 512, + 512 + ], + "empty_cameras": 0, + "adapt_to_pi_aloha": false, + "use_delta_joint_actions_aloha": false, + "tokenizer_max_length": 48, + "num_steps": 10, + "use_cache": true, + "freeze_vision_encoder": true, + "train_expert_only": true, + "train_state_proj": true, + "optimizer_lr": 2e-05, + "optimizer_betas": [ + 0.9, + 0.95 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 0.01, + "optimizer_grad_clip_norm": 10.0, + "scheduler_warmup_steps": 1000, + "scheduler_decay_steps": 30000, + "scheduler_decay_lr": 2.5e-06, + "vlm_model_name": "HuggingFaceTB/SmolVLM2-500M-Video-Instruct", + "load_vlm_weights": true, + "add_image_special_tokens": false, + "attention_mode": "cross_attn", + "prefix_length": 0, + "pad_language_to": "max_length", + "num_expert_layers": 0, + "num_vlm_layers": 16, + "self_attn_every_n_layers": 2, + "expert_width_multiplier": 0.75, + "min_period": 0.004, + "max_period": 4.0, + "rtc_config": null, + "compile_model": false, + "compile_mode": "max-autotune" + }, + "output_dir": "outputs/train/sequential_t1_task48_fft", + "job_name": "sequential_t1_task48_fft", + "resume": false, + "seed": 42, + "cudnn_deterministic": false, + "num_workers": 4, + "batch_size": 16, + "steps": 3000, + "eval_freq": 0, + "log_freq": 1, + "tolerance_s": 0.0001, + "save_checkpoint": true, + "save_freq": 500, + "use_policy_training_preset": true, + "optimizer": { + "type": "adamw", + "lr": 2e-05, + "weight_decay": 0.01, + "grad_clip_norm": 10.0, + "betas": [ + 0.9, + 0.95 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "cosine_decay_with_warmup", + "num_warmup_steps": 1000, + "num_decay_steps": 30000, + "peak_lr": 2e-05, + "decay_lr": 2.5e-06 + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": true, + "project": "lerobot-smolvla", + "entity": null, + "notes": null, + "run_id": "ooipb6on", + "mode": null, + "add_tags": true + }, + "peft": null, + "use_rabc": false, + "rabc_progress_path": null, + "rabc_kappa": 0.01, + "rabc_epsilon": 1e-06, + "rabc_head_mode": "sparse", + "rename_map": {}, + "checkpoint_path": null +} \ No newline at end of file diff --git a/checkpoints/002500/training_state/optimizer_param_groups.json b/checkpoints/002500/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..6088944190cc469301ca82b90b718e0ccf8b8e25 --- /dev/null +++ b/checkpoints/002500/training_state/optimizer_param_groups.json @@ -0,0 +1,521 @@ +[ + { + "lr": 3.6722777168861617e-06, + "betas": [ + 0.9, + 0.95 + ], + "eps": 1e-08, + "weight_decay": 0.01, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "initial_lr": 2e-05, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310, + 311, + 312, + 313, + 314, + 315, + 316, + 317, + 318, + 319, + 320, + 321, + 322, + 323, + 324, + 325, + 326, + 327, + 328, + 329, + 330, + 331, + 332, + 333, + 334, + 335, + 336, + 337, + 338, + 339, + 340, + 341, + 342, + 343, + 344, + 345, + 346, + 347, + 348, + 349, + 350, + 351, + 352, + 353, + 354, + 355, + 356, + 357, + 358, + 359, + 360, + 361, + 362, + 363, + 364, + 365, + 366, + 367, + 368, + 369, + 370, + 371, + 372, + 373, + 374, + 375, + 376, + 377, + 378, + 379, + 380, + 381, + 382, + 383, + 384, + 385, + 386, + 387, + 388, + 389, + 390, + 391, + 392, + 393, + 394, + 395, + 396, + 397, + 398, + 399, + 400, + 401, + 402, + 403, + 404, + 405, + 406, + 407, + 408, + 409, + 410, + 411, + 412, + 413, + 414, + 415, + 416, + 417, + 418, + 419, + 420, + 421, + 422, + 423, + 424, + 425, + 426, + 427, + 428, + 429, + 430, + 431, + 432, + 433, + 434, + 435, + 436, + 437, + 438, + 439, + 440, + 441, + 442, + 443, + 444, + 445, + 446, + 447, + 448, + 449, + 450, + 451, + 452, + 453, + 454, + 455, + 456, + 457, + 458, + 459, + 460, + 461, + 462, + 463, + 464, + 465, + 466, + 467, + 468, + 469, + 470, + 471, + 472, + 473, + 474, + 475, + 476, + 477, + 478, + 479, + 480, + 481, + 482, + 483, + 484, + 485, + 486, + 487, + 488, + 489, + 490, + 491, + 492, + 493, + 494, + 495, + 496, + 497, + 498, + 499 + ] + } +] \ No newline at end of file diff --git a/checkpoints/002500/training_state/optimizer_state.safetensors b/checkpoints/002500/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c26095038c3ba5c97c3e60070cb0dc11b36c8c58 --- /dev/null +++ b/checkpoints/002500/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0154094175112d20b92a1bdde6a4b64d00e20d7dbe62cc0286b688cb6405713 +size 412659164 diff --git a/checkpoints/002500/training_state/rng_state.safetensors b/checkpoints/002500/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..6e46a1129cdb08bf7e58086eb79825bb80bacc08 --- /dev/null +++ b/checkpoints/002500/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2bcbc17ca34b2fb85cb49c836856efbe2b2aea855bf93082708eacdbb6eccaa3 +size 15708 diff --git a/checkpoints/002500/training_state/scheduler_state.json b/checkpoints/002500/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..4931bf560369418987dc4217758025f619701dd3 --- /dev/null +++ b/checkpoints/002500/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 2e-05 + ], + "last_epoch": 2500, + "_step_count": 2501, + "_is_initial": false, + "_get_lr_called_within_step": false, + "_last_lr": [ + 3.6722777168861617e-06 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/checkpoints/002500/training_state/training_step.json b/checkpoints/002500/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..d265f47a09c2910099ed59e197b57b34675d1ae0 --- /dev/null +++ b/checkpoints/002500/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 2500 +} \ No newline at end of file diff --git a/checkpoints/003000/pretrained_model/config.json b/checkpoints/003000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..3bdae2d4dfa4b23cad255243a126b98d8f5d62c6 --- /dev/null +++ b/checkpoints/003000/pretrained_model/config.json @@ -0,0 +1,111 @@ +{ + "type": "smolvla", + "n_obs_steps": 1, + "input_features": { + "observation.images.image": { + "type": "VISUAL", + "shape": [ + 256, + 256, + 3 + ] + }, + "observation.images.wrist_image": { + "type": "VISUAL", + "shape": [ + 256, + 256, + 3 + ] + }, + "observation.state": { + "type": "STATE", + "shape": [ + 8 + ] + }, + "observation.states.ee_state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.states.joint_state": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "observation.states.gripper_state": { + "type": "STATE", + "shape": [ + 2 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "device": "cuda", + "use_amp": false, + "use_peft": false, + "push_to_hub": true, + "repo_id": "ardalon/sequential_t1_task48_fft", + "private": null, + "tags": null, + "license": null, + "pretrained_path": "/content/smolvla_base", + "chunk_size": 50, + "n_action_steps": 50, + "normalization_mapping": { + "VISUAL": "IDENTITY", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "max_state_dim": 32, + "max_action_dim": 32, + "resize_imgs_with_padding": [ + 512, + 512 + ], + "empty_cameras": 0, + "adapt_to_pi_aloha": false, + "use_delta_joint_actions_aloha": false, + "tokenizer_max_length": 48, + "num_steps": 10, + "use_cache": true, + "freeze_vision_encoder": true, + "train_expert_only": true, + "train_state_proj": true, + "optimizer_lr": 2e-05, + "optimizer_betas": [ + 0.9, + 0.95 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 0.01, + "optimizer_grad_clip_norm": 10.0, + "scheduler_warmup_steps": 1000, + "scheduler_decay_steps": 30000, + "scheduler_decay_lr": 2.5e-06, + "vlm_model_name": "HuggingFaceTB/SmolVLM2-500M-Video-Instruct", + "load_vlm_weights": true, + "add_image_special_tokens": false, + "attention_mode": "cross_attn", + "prefix_length": 0, + "pad_language_to": "max_length", + "num_expert_layers": 0, + "num_vlm_layers": 16, + "self_attn_every_n_layers": 2, + "expert_width_multiplier": 0.75, + "min_period": 0.004, + "max_period": 4.0, + "rtc_config": null, + "compile_model": false, + "compile_mode": "max-autotune" +} \ No newline at end of file diff --git a/checkpoints/003000/pretrained_model/model.safetensors b/checkpoints/003000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..56375b1a356e957a1b21a1244c14f3abea592ead --- /dev/null +++ b/checkpoints/003000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3cecc83df49e006d9f235ddf486ff82d3b3bf1372ad91b9ff9c0bd335903e0da +size 906712520 diff --git a/checkpoints/003000/pretrained_model/policy_postprocessor.json b/checkpoints/003000/pretrained_model/policy_postprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..0ec8146a07f9b92bc5a3c02e2d6b7d49ba6d26e8 --- /dev/null +++ b/checkpoints/003000/pretrained_model/policy_postprocessor.json @@ -0,0 +1,32 @@ +{ + "name": "policy_postprocessor", + "steps": [ + { + "registry_name": "unnormalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "norm_map": { + "VISUAL": "IDENTITY", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + } + }, + "state_file": "policy_postprocessor_step_0_unnormalizer_processor.safetensors" + }, + { + "registry_name": "device_processor", + "config": { + "device": "cpu", + "float_dtype": null + } + } + ] +} \ No newline at end of file diff --git a/checkpoints/003000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors b/checkpoints/003000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..fb3b6bb870bf669cd89d6bc4600478c768d22459 --- /dev/null +++ b/checkpoints/003000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2abc5a6a77ada2e972696d0b4da9485e8ac2c2036629207a51cf587b2c434abb +size 5440 diff --git a/checkpoints/003000/pretrained_model/policy_preprocessor.json b/checkpoints/003000/pretrained_model/policy_preprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..7527b68cb0a2d7c7a1a0f858ac44bec3df5ac0bf --- /dev/null +++ b/checkpoints/003000/pretrained_model/policy_preprocessor.json @@ -0,0 +1,97 @@ +{ + "name": "policy_preprocessor", + "steps": [ + { + "registry_name": "rename_observations_processor", + "config": { + "rename_map": {} + } + }, + { + "registry_name": "to_batch_processor", + "config": {} + }, + { + "registry_name": "smolvla_new_line_processor", + "config": {} + }, + { + "registry_name": "tokenizer_processor", + "config": { + "max_length": 48, + "task_key": "task", + "padding_side": "right", + "padding": "max_length", + "truncation": true, + "tokenizer_name": "HuggingFaceTB/SmolVLM2-500M-Video-Instruct" + } + }, + { + "registry_name": "device_processor", + "config": { + "device": "cuda", + "float_dtype": null + } + }, + { + "registry_name": "normalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "observation.images.image": { + "type": "VISUAL", + "shape": [ + 256, + 256, + 3 + ] + }, + "observation.images.wrist_image": { + "type": "VISUAL", + "shape": [ + 256, + 256, + 3 + ] + }, + "observation.state": { + "type": "STATE", + "shape": [ + 8 + ] + }, + "observation.states.ee_state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.states.joint_state": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "observation.states.gripper_state": { + "type": "STATE", + "shape": [ + 2 + ] + }, + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "norm_map": { + "VISUAL": "IDENTITY", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + } + }, + "state_file": "policy_preprocessor_step_5_normalizer_processor.safetensors" + } + ] +} \ No newline at end of file diff --git a/checkpoints/003000/pretrained_model/policy_preprocessor_step_5_normalizer_processor.safetensors b/checkpoints/003000/pretrained_model/policy_preprocessor_step_5_normalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..fb3b6bb870bf669cd89d6bc4600478c768d22459 --- /dev/null +++ b/checkpoints/003000/pretrained_model/policy_preprocessor_step_5_normalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2abc5a6a77ada2e972696d0b4da9485e8ac2c2036629207a51cf587b2c434abb +size 5440 diff --git a/checkpoints/003000/pretrained_model/train_config.json b/checkpoints/003000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..06c696e54ad5d01142c45fe44d4d30291256fb50 --- /dev/null +++ b/checkpoints/003000/pretrained_model/train_config.json @@ -0,0 +1,392 @@ +{ + "dataset": { + "repo_id": "daniecraig/libero_90_no_noops_lerobot_v30", + "root": null, + "episodes": [ + 3092, + 3093, + 3094, + 3095, + 3096, + 3097, + 3098, + 3099, + 3100, + 3101, + 3102, + 3103, + 3104, + 3105, + 3106, + 3107, + 3108, + 3109, + 3110, + 3111, + 3112, + 3113, + 3114, + 3115, + 3116, + 3117, + 3118, + 3119, + 3120, + 3121, + 3122, + 3123, + 3124, + 3125, + 3126, + 3127, + 3128, + 3129, + 3130, + 3131, + 3132, + 3133, + 3134, + 3135, + 3136 + ], + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + }, + "affine": { + "weight": 1.0, + "type": "RandomAffine", + "kwargs": { + "degrees": [ + -5.0, + 5.0 + ], + "translate": [ + 0.05, + 0.05 + ] + } + } + } + }, + "revision": "main", + "use_imagenet_stats": true, + "video_backend": "torchcodec", + "streaming": false + }, + "env": { + "type": "libero", + "task": "libero_90", + "fps": 30, + "features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + }, + "pixels/agentview_image": { + "type": "VISUAL", + "shape": [ + 360, + 360, + 3 + ] + }, + "pixels/robot0_eye_in_hand_image": { + "type": "VISUAL", + "shape": [ + 360, + 360, + 3 + ] + }, + "robot_state/eef/pos": { + "type": "STATE", + "shape": [ + 3 + ] + }, + "robot_state/eef/quat": { + "type": "STATE", + "shape": [ + 4 + ] + }, + "robot_state/eef/mat": { + "type": "STATE", + "shape": [ + 3, + 3 + ] + }, + "robot_state/gripper/qpos": { + "type": "STATE", + "shape": [ + 2 + ] + }, + "robot_state/gripper/qvel": { + "type": "STATE", + "shape": [ + 2 + ] + }, + "robot_state/joints/pos": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "robot_state/joints/vel": { + "type": "STATE", + "shape": [ + 7 + ] + } + }, + "features_map": { + "action": "action", + "robot_state/eef/pos": "observation.state.eef_pos", + "robot_state/eef/quat": "observation.state.eef_quat", + "robot_state/eef/mat": "observation.state.eef_mat", + "robot_state/gripper/qpos": "observation.state.gripper_qpos", + "robot_state/gripper/qvel": "observation.state.gripper_qvel", + "robot_state/joints/pos": "observation.state.joint_pos", + "robot_state/joints/vel": "observation.state.joint_vel", + "pixels/agentview_image": "observation.images.image", + "pixels/robot0_eye_in_hand_image": "observation.images.image2" + }, + "max_parallel_tasks": 1, + "disable_env_checker": true, + "task_ids": [ + 48 + ], + "episode_length": null, + "obs_type": "pixels_agent_pos", + "render_mode": "rgb_array", + "camera_name": "agentview_image,robot0_eye_in_hand_image", + "init_states": true, + "camera_name_mapping": null, + "observation_height": 360, + "observation_width": 360, + "control_mode": "relative" + }, + "policy": { + "type": "smolvla", + "n_obs_steps": 1, + "input_features": { + "observation.images.image": { + "type": "VISUAL", + "shape": [ + 256, + 256, + 3 + ] + }, + "observation.images.wrist_image": { + "type": "VISUAL", + "shape": [ + 256, + 256, + 3 + ] + }, + "observation.state": { + "type": "STATE", + "shape": [ + 8 + ] + }, + "observation.states.ee_state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.states.joint_state": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "observation.states.gripper_state": { + "type": "STATE", + "shape": [ + 2 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "device": "cuda", + "use_amp": false, + "use_peft": false, + "push_to_hub": true, + "repo_id": "ardalon/sequential_t1_task48_fft", + "private": null, + "tags": null, + "license": null, + "pretrained_path": "/content/smolvla_base", + "chunk_size": 50, + "n_action_steps": 50, + "normalization_mapping": { + "VISUAL": "IDENTITY", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "max_state_dim": 32, + "max_action_dim": 32, + "resize_imgs_with_padding": [ + 512, + 512 + ], + "empty_cameras": 0, + "adapt_to_pi_aloha": false, + "use_delta_joint_actions_aloha": false, + "tokenizer_max_length": 48, + "num_steps": 10, + "use_cache": true, + "freeze_vision_encoder": true, + "train_expert_only": true, + "train_state_proj": true, + "optimizer_lr": 2e-05, + "optimizer_betas": [ + 0.9, + 0.95 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 0.01, + "optimizer_grad_clip_norm": 10.0, + "scheduler_warmup_steps": 1000, + "scheduler_decay_steps": 30000, + "scheduler_decay_lr": 2.5e-06, + "vlm_model_name": "HuggingFaceTB/SmolVLM2-500M-Video-Instruct", + "load_vlm_weights": true, + "add_image_special_tokens": false, + "attention_mode": "cross_attn", + "prefix_length": 0, + "pad_language_to": "max_length", + "num_expert_layers": 0, + "num_vlm_layers": 16, + "self_attn_every_n_layers": 2, + "expert_width_multiplier": 0.75, + "min_period": 0.004, + "max_period": 4.0, + "rtc_config": null, + "compile_model": false, + "compile_mode": "max-autotune" + }, + "output_dir": "outputs/train/sequential_t1_task48_fft", + "job_name": "sequential_t1_task48_fft", + "resume": false, + "seed": 42, + "cudnn_deterministic": false, + "num_workers": 4, + "batch_size": 16, + "steps": 3000, + "eval_freq": 0, + "log_freq": 1, + "tolerance_s": 0.0001, + "save_checkpoint": true, + "save_freq": 500, + "use_policy_training_preset": true, + "optimizer": { + "type": "adamw", + "lr": 2e-05, + "weight_decay": 0.01, + "grad_clip_norm": 10.0, + "betas": [ + 0.9, + 0.95 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "cosine_decay_with_warmup", + "num_warmup_steps": 1000, + "num_decay_steps": 30000, + "peak_lr": 2e-05, + "decay_lr": 2.5e-06 + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": true, + "project": "lerobot-smolvla", + "entity": null, + "notes": null, + "run_id": "ooipb6on", + "mode": null, + "add_tags": true + }, + "peft": null, + "use_rabc": false, + "rabc_progress_path": null, + "rabc_kappa": 0.01, + "rabc_epsilon": 1e-06, + "rabc_head_mode": "sparse", + "rename_map": {}, + "checkpoint_path": null +} \ No newline at end of file diff --git a/checkpoints/003000/training_state/optimizer_param_groups.json b/checkpoints/003000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..a0609ea2fafd807fbd4faa8d694128c881a4b1e9 --- /dev/null +++ b/checkpoints/003000/training_state/optimizer_param_groups.json @@ -0,0 +1,521 @@ +[ + { + "lr": 2.5e-06, + "betas": [ + 0.9, + 0.95 + ], + "eps": 1e-08, + "weight_decay": 0.01, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "initial_lr": 2e-05, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310, + 311, + 312, + 313, + 314, + 315, + 316, + 317, + 318, + 319, + 320, + 321, + 322, + 323, + 324, + 325, + 326, + 327, + 328, + 329, + 330, + 331, + 332, + 333, + 334, + 335, + 336, + 337, + 338, + 339, + 340, + 341, + 342, + 343, + 344, + 345, + 346, + 347, + 348, + 349, + 350, + 351, + 352, + 353, + 354, + 355, + 356, + 357, + 358, + 359, + 360, + 361, + 362, + 363, + 364, + 365, + 366, + 367, + 368, + 369, + 370, + 371, + 372, + 373, + 374, + 375, + 376, + 377, + 378, + 379, + 380, + 381, + 382, + 383, + 384, + 385, + 386, + 387, + 388, + 389, + 390, + 391, + 392, + 393, + 394, + 395, + 396, + 397, + 398, + 399, + 400, + 401, + 402, + 403, + 404, + 405, + 406, + 407, + 408, + 409, + 410, + 411, + 412, + 413, + 414, + 415, + 416, + 417, + 418, + 419, + 420, + 421, + 422, + 423, + 424, + 425, + 426, + 427, + 428, + 429, + 430, + 431, + 432, + 433, + 434, + 435, + 436, + 437, + 438, + 439, + 440, + 441, + 442, + 443, + 444, + 445, + 446, + 447, + 448, + 449, + 450, + 451, + 452, + 453, + 454, + 455, + 456, + 457, + 458, + 459, + 460, + 461, + 462, + 463, + 464, + 465, + 466, + 467, + 468, + 469, + 470, + 471, + 472, + 473, + 474, + 475, + 476, + 477, + 478, + 479, + 480, + 481, + 482, + 483, + 484, + 485, + 486, + 487, + 488, + 489, + 490, + 491, + 492, + 493, + 494, + 495, + 496, + 497, + 498, + 499 + ] + } +] \ No newline at end of file diff --git a/checkpoints/003000/training_state/optimizer_state.safetensors b/checkpoints/003000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5ad1e53b6e9da61d3be417ed49b82dcd8eb29464 --- /dev/null +++ b/checkpoints/003000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d757290cccb94006240b0e08dc9153312867e440a91eb6e7653fa8bb90a45a0 +size 412659164 diff --git a/checkpoints/003000/training_state/rng_state.safetensors b/checkpoints/003000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d235488c83e5f3de72b696d71611492b5f9aa956 --- /dev/null +++ b/checkpoints/003000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cbd415e3cb4605ec14dc796629a6845edb18d294baf09d5036de8887041b819b +size 15708 diff --git a/checkpoints/003000/training_state/scheduler_state.json b/checkpoints/003000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..eadb41e983dfc81823cc44f7c06c900abb00bc99 --- /dev/null +++ b/checkpoints/003000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 2e-05 + ], + "last_epoch": 3000, + "_step_count": 3001, + "_is_initial": false, + "_get_lr_called_within_step": false, + "_last_lr": [ + 2.5e-06 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/checkpoints/003000/training_state/training_step.json b/checkpoints/003000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..c1a44127b7dfea653fd776d529fa83c55d32081c --- /dev/null +++ b/checkpoints/003000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 3000 +} \ No newline at end of file diff --git a/wandb/debug-internal.log b/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..323e2c9019268287c816a981c31bcc11e4b2d7e9 --- /dev/null +++ b/wandb/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2026-04-08T06:11:52.655319135Z","level":"INFO","msg":"stream: starting","core version":"0.24.2"} +{"time":"2026-04-08T06:11:52.844348169Z","level":"INFO","msg":"stream: created new stream","id":"ooipb6on"} +{"time":"2026-04-08T06:11:52.844423175Z","level":"INFO","msg":"handler: started","stream_id":"ooipb6on"} +{"time":"2026-04-08T06:11:52.844565623Z","level":"INFO","msg":"stream: started","id":"ooipb6on"} +{"time":"2026-04-08T06:11:52.844576184Z","level":"INFO","msg":"writer: started","stream_id":"ooipb6on"} +{"time":"2026-04-08T06:11:52.844627437Z","level":"INFO","msg":"sender: started","stream_id":"ooipb6on"} +{"time":"2026-04-08T06:24:24.596553906Z","level":"INFO","msg":"stream: closing","id":"ooipb6on"} +{"time":"2026-04-08T06:24:24.764235068Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-04-08T06:24:24.889363837Z","level":"INFO","msg":"handler: closed","stream_id":"ooipb6on"} +{"time":"2026-04-08T06:24:24.889501554Z","level":"INFO","msg":"sender: closed","stream_id":"ooipb6on"} +{"time":"2026-04-08T06:24:24.889521676Z","level":"INFO","msg":"stream: closed","id":"ooipb6on"} diff --git a/wandb/debug.log b/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..b6a0edb1281f69bbb58309b061c5cb716ae4f522 --- /dev/null +++ b/wandb/debug.log @@ -0,0 +1,21 @@ +2026-04-08 06:11:52,307 INFO MainThread:2408 [wandb_setup.py:_flush():81] Current SDK version is 0.24.2 +2026-04-08 06:11:52,307 INFO MainThread:2408 [wandb_setup.py:_flush():81] Configure stats pid to 2408 +2026-04-08 06:11:52,307 INFO MainThread:2408 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-04-08 06:11:52,307 INFO MainThread:2408 [wandb_init.py:setup_run_log_directory():717] Logging user logs to outputs/train/sequential_t1_task48_fft/wandb/run-20260408_061152-ooipb6on/logs/debug.log +2026-04-08 06:11:52,307 INFO MainThread:2408 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to outputs/train/sequential_t1_task48_fft/wandb/run-20260408_061152-ooipb6on/logs/debug-internal.log +2026-04-08 06:11:52,307 INFO MainThread:2408 [wandb_init.py:init():844] calling init triggers +2026-04-08 06:11:52,307 INFO MainThread:2408 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'dataset': {'repo_id': 'daniecraig/libero_90_no_noops_lerobot_v30', 'root': None, 'episodes': [3092, 3093, 3094, 3095, 3096, 3097, 3098, 3099, 3100, 3101, 3102, 3103, 3104, 3105, 3106, 3107, 3108, 3109, 3110, 3111, 3112, 3113, 3114, 3115, 3116, 3117, 3118, 3119, 3120, 3121, 3122, 3123, 3124, 3125, 3126, 3127, 3128, 3129, 3130, 3131, 3132, 3133, 3134, 3135, 3136], 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': 'main', 'use_imagenet_stats': True, 'video_backend': 'torchcodec', 'streaming': False}, 'env': {'type': 'libero', 'task': 'libero_90', 'fps': 30, 'features': {'action': {'type': , 'shape': [7]}, 'pixels/agentview_image': {'type': , 'shape': [360, 360, 3]}, 'pixels/robot0_eye_in_hand_image': {'type': , 'shape': [360, 360, 3]}, 'robot_state/eef/pos': {'type': , 'shape': [3]}, 'robot_state/eef/quat': {'type': , 'shape': [4]}, 'robot_state/eef/mat': {'type': , 'shape': [3, 3]}, 'robot_state/gripper/qpos': {'type': , 'shape': [2]}, 'robot_state/gripper/qvel': {'type': , 'shape': [2]}, 'robot_state/joints/pos': {'type': , 'shape': [7]}, 'robot_state/joints/vel': {'type': , 'shape': [7]}}, 'features_map': {'action': 'action', 'robot_state/eef/pos': 'observation.state.eef_pos', 'robot_state/eef/quat': 'observation.state.eef_quat', 'robot_state/eef/mat': 'observation.state.eef_mat', 'robot_state/gripper/qpos': 'observation.state.gripper_qpos', 'robot_state/gripper/qvel': 'observation.state.gripper_qvel', 'robot_state/joints/pos': 'observation.state.joint_pos', 'robot_state/joints/vel': 'observation.state.joint_vel', 'pixels/agentview_image': 'observation.images.image', 'pixels/robot0_eye_in_hand_image': 'observation.images.image2'}, 'max_parallel_tasks': 1, 'disable_env_checker': True, 'task_ids': [48], 'episode_length': None, 'obs_type': 'pixels_agent_pos', 'render_mode': 'rgb_array', 'camera_name': 'agentview_image,robot0_eye_in_hand_image', 'init_states': True, 'camera_name_mapping': None, 'observation_height': 360, 'observation_width': 360, 'control_mode': 'relative'}, 'policy': {'type': 'smolvla', 'n_obs_steps': 1, 'input_features': None, 'output_features': None, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': True, 'repo_id': 'ardalon/sequential_t1_task48_fft', 'private': None, 'tags': None, 'license': None, 'pretrained_path': '/content/smolvla_base', 'chunk_size': 50, 'n_action_steps': 50, 'normalization_mapping': {'VISUAL': , 'STATE': , 'ACTION': }, 'max_state_dim': 32, 'max_action_dim': 32, 'resize_imgs_with_padding': [512, 512], 'empty_cameras': 0, 'adapt_to_pi_aloha': False, 'use_delta_joint_actions_aloha': False, 'tokenizer_max_length': 48, 'num_steps': 10, 'use_cache': True, 'freeze_vision_encoder': True, 'train_expert_only': True, 'train_state_proj': True, 'optimizer_lr': 2e-05, 'optimizer_betas': [0.9, 0.95], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 0.01, 'optimizer_grad_clip_norm': 10.0, 'scheduler_warmup_steps': 1000, 'scheduler_decay_steps': 30000, 'scheduler_decay_lr': 2.5e-06, 'vlm_model_name': 'HuggingFaceTB/SmolVLM2-500M-Video-Instruct', 'load_vlm_weights': True, 'add_image_special_tokens': False, 'attention_mode': 'cross_attn', 'prefix_length': 0, 'pad_language_to': 'max_length', 'num_expert_layers': 0, 'num_vlm_layers': 16, 'self_attn_every_n_layers': 2, 'expert_width_multiplier': 0.75, 'min_period': 0.004, 'max_period': 4.0, 'rtc_config': None, 'compile_model': False, 'compile_mode': 'max-autotune'}, 'output_dir': 'outputs/train/sequential_t1_task48_fft', 'job_name': 'sequential_t1_task48_fft', 'resume': False, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 16, 'steps': 3000, 'eval_freq': 0, 'log_freq': 1, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 500, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 2e-05, 'weight_decay': 0.01, 'grad_clip_norm': 10.0, 'betas': [0.9, 0.95], 'eps': 1e-08}, 'scheduler': {'type': 'cosine_decay_with_warmup', 'num_warmup_steps': 1000, 'num_decay_steps': 30000, 'peak_lr': 2e-05, 'decay_lr': 2.5e-06}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot-smolvla', 'entity': None, 'notes': None, 'run_id': None, 'mode': None, 'add_tags': True}, 'peft': None, 'use_rabc': False, 'rabc_progress_path': None, 'rabc_kappa': 0.01, 'rabc_epsilon': 1e-06, 'rabc_head_mode': 'sparse', 'rename_map': {}, 'checkpoint_path': None, '_wandb': {}} +2026-04-08 06:11:52,307 INFO MainThread:2408 [wandb_init.py:init():892] starting backend +2026-04-08 06:11:52,644 INFO MainThread:2408 [wandb_init.py:init():895] sending inform_init request +2026-04-08 06:11:52,653 INFO MainThread:2408 [wandb_init.py:init():903] backend started and connected +2026-04-08 06:11:52,656 INFO MainThread:2408 [wandb_init.py:init():973] updated telemetry +2026-04-08 06:11:52,657 INFO MainThread:2408 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-04-08 06:11:53,136 INFO MainThread:2408 [wandb_init.py:init():1042] starting run threads in backend +2026-04-08 06:11:53,673 INFO MainThread:2408 [wandb_run.py:_console_start():2529] atexit reg +2026-04-08 06:11:53,673 INFO MainThread:2408 [wandb_run.py:_redirect():2377] redirect: wrap_raw +2026-04-08 06:11:53,673 INFO MainThread:2408 [wandb_run.py:_redirect():2446] Wrapping output streams. +2026-04-08 06:11:53,673 INFO MainThread:2408 [wandb_run.py:_redirect():2469] Redirects installed. +2026-04-08 06:11:53,677 INFO MainThread:2408 [wandb_init.py:init():1082] run started, returning control to user process +2026-04-08 06:24:24,596 INFO wandb-AsyncioManager-main:2408 [service_client.py:_forward_responses():94] Reached EOF. +2026-04-08 06:24:24,596 INFO wandb-AsyncioManager-main:2408 [mailbox.py:close():154] Closing mailbox, abandoning 1 handles. diff --git a/wandb/run-20260408_061152-ooipb6on/files/config.yaml b/wandb/run-20260408_061152-ooipb6on/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2b38e233a52cd2fdcd457f7870ad7d24e3335450 --- /dev/null +++ b/wandb/run-20260408_061152-ooipb6on/files/config.yaml @@ -0,0 +1,407 @@ +_wandb: + value: + cli_version: 0.24.2 + e: + wwf5oly56zhuth2ptlhge7gx9ob3nvbg: + args: + - --policy.path=/content/smolvla_base + - --policy.input_features=null + - --policy.output_features=null + - --policy.device=cuda + - --dataset.repo_id=daniecraig/libero_90_no_noops_lerobot_v30 + - --dataset.revision=main + - --dataset.episodes=[3092, 3093, 3094, 3095, 3096, 3097, 3098, 3099, 3100, 3101, 3102, 3103, 3104, 3105, 3106, 3107, 3108, 3109, 3110, 3111, 3112, 3113, 3114, 3115, 3116, 3117, 3118, 3119, 3120, 3121, 3122, 3123, 3124, 3125, 3126, 3127, 3128, 3129, 3130, 3131, 3132, 3133, 3134, 3135, 3136] + - --env.type=libero + - --env.task=libero_90 + - --env.task_ids=[48] + - --env.control_mode=relative + - --batch_size=16 + - --eval_freq=0 + - --save_freq=500 + - --seed=42 + - --output_dir=outputs/train/sequential_t1_task48_fft + - --policy.optimizer_lr=2e-5 + - --policy.optimizer_weight_decay=0.01 + - --steps=3000 + - --job_name=sequential_t1_task48_fft + - --wandb.enable=true + - --wandb.project=lerobot-smolvla + - --wandb.disable_artifac=true + - --policy.push_to_hub=true + - --policy.repo_id=ardalon/sequential_t1_task48_fft + - --log_freq=1 + cpu_count: 6 + cpu_count_logical: 12 + cudaVersion: "13.0" + disk: + /: + total: "253055008768" + used: "49321058304" + executable: /usr/bin/python3 + gpu: NVIDIA A100-SXM4-40GB + gpu_count: 1 + gpu_nvidia: + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "42949672960" + name: NVIDIA A100-SXM4-40GB + uuid: GPU-8127346a-b290-0d4a-3e6a-5d346abf9d4b + host: 01748ae5acaf + memory: + total: "89629200384" + os: Linux-6.6.113+-x86_64-with-glibc2.35 + program: /usr/local/bin/lerobot-train + python: CPython 3.12.13 + root: outputs/train/sequential_t1_task48_fft + startedAt: "2026-04-08T06:11:52.305836Z" + writerId: wwf5oly56zhuth2ptlhge7gx9ob3nvbg + m: [] + python_version: 3.12.13 + t: + "1": + - 1 + - 5 + - 11 + - 41 + - 49 + - 51 + - 53 + - 71 + - 83 + - 105 + "2": + - 1 + - 5 + - 11 + - 41 + - 49 + - 51 + - 53 + - 71 + - 83 + - 105 + "3": + - 13 + - 15 + - 16 + - 61 + "4": 3.12.13 + "5": 0.24.2 + "6": 5.3.0 + "10": + - 21 + "12": 0.24.2 + "13": linux-x86_64 +batch_size: + value: 16 +checkpoint_path: + value: null +cudnn_deterministic: + value: false +dataset: + value: + episodes: + - 3092 + - 3093 + - 3094 + - 3095 + - 3096 + - 3097 + - 3098 + - 3099 + - 3100 + - 3101 + - 3102 + - 3103 + - 3104 + - 3105 + - 3106 + - 3107 + - 3108 + - 3109 + - 3110 + - 3111 + - 3112 + - 3113 + - 3114 + - 3115 + - 3116 + - 3117 + - 3118 + - 3119 + - 3120 + - 3121 + - 3122 + - 3123 + - 3124 + - 3125 + - 3126 + - 3127 + - 3128 + - 3129 + - 3130 + - 3131 + - 3132 + - 3133 + - 3134 + - 3135 + - 3136 + image_transforms: + enable: false + max_num_transforms: 3 + random_order: false + tfs: + affine: + kwargs: + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + type: RandomAffine + weight: 1 + brightness: + kwargs: + brightness: + - 0.8 + - 1.2 + type: ColorJitter + weight: 1 + contrast: + kwargs: + contrast: + - 0.8 + - 1.2 + type: ColorJitter + weight: 1 + hue: + kwargs: + hue: + - -0.05 + - 0.05 + type: ColorJitter + weight: 1 + saturation: + kwargs: + saturation: + - 0.5 + - 1.5 + type: ColorJitter + weight: 1 + sharpness: + kwargs: + sharpness: + - 0.5 + - 1.5 + type: SharpnessJitter + weight: 1 + repo_id: daniecraig/libero_90_no_noops_lerobot_v30 + revision: main + root: null + streaming: false + use_imagenet_stats: true + video_backend: torchcodec +env: + value: + camera_name: agentview_image,robot0_eye_in_hand_image + camera_name_mapping: null + control_mode: relative + disable_env_checker: true + episode_length: null + features: + action: + shape: + - 7 + type: ACTION + pixels/agentview_image: + shape: + - 360 + - 360 + - 3 + type: VISUAL + pixels/robot0_eye_in_hand_image: + shape: + - 360 + - 360 + - 3 + type: VISUAL + robot_state/eef/mat: + shape: + - 3 + - 3 + type: STATE + robot_state/eef/pos: + shape: + - 3 + type: STATE + robot_state/eef/quat: + shape: + - 4 + type: STATE + robot_state/gripper/qpos: + shape: + - 2 + type: STATE + robot_state/gripper/qvel: + shape: + - 2 + type: STATE + robot_state/joints/pos: + shape: + - 7 + type: STATE + robot_state/joints/vel: + shape: + - 7 + type: STATE + features_map: + action: action + pixels/agentview_image: observation.images.image + pixels/robot0_eye_in_hand_image: observation.images.image2 + robot_state/eef/mat: observation.state.eef_mat + robot_state/eef/pos: observation.state.eef_pos + robot_state/eef/quat: observation.state.eef_quat + robot_state/gripper/qpos: observation.state.gripper_qpos + robot_state/gripper/qvel: observation.state.gripper_qvel + robot_state/joints/pos: observation.state.joint_pos + robot_state/joints/vel: observation.state.joint_vel + fps: 30 + init_states: true + max_parallel_tasks: 1 + obs_type: pixels_agent_pos + observation_height: 360 + observation_width: 360 + render_mode: rgb_array + task: libero_90 + task_ids: + - 48 + type: libero +eval: + value: + batch_size: 50 + n_episodes: 50 + use_async_envs: false +eval_freq: + value: 0 +job_name: + value: sequential_t1_task48_fft +log_freq: + value: 1 +num_workers: + value: 4 +optimizer: + value: + betas: + - 0.9 + - 0.95 + eps: 1e-08 + grad_clip_norm: 10 + lr: 2e-05 + type: adamw + weight_decay: 0.01 +output_dir: + value: outputs/train/sequential_t1_task48_fft +peft: + value: null +policy: + value: + adapt_to_pi_aloha: false + add_image_special_tokens: false + attention_mode: cross_attn + chunk_size: 50 + compile_mode: max-autotune + compile_model: false + device: cuda + empty_cameras: 0 + expert_width_multiplier: 0.75 + freeze_vision_encoder: true + input_features: null + license: null + load_vlm_weights: true + max_action_dim: 32 + max_period: 4 + max_state_dim: 32 + min_period: 0.004 + n_action_steps: 50 + n_obs_steps: 1 + normalization_mapping: + ACTION: MEAN_STD + STATE: MEAN_STD + VISUAL: IDENTITY + num_expert_layers: 0 + num_steps: 10 + num_vlm_layers: 16 + optimizer_betas: + - 0.9 + - 0.95 + optimizer_eps: 1e-08 + optimizer_grad_clip_norm: 10 + optimizer_lr: 2e-05 + optimizer_weight_decay: 0.01 + output_features: null + pad_language_to: max_length + prefix_length: 0 + pretrained_path: /content/smolvla_base + private: null + push_to_hub: true + repo_id: ardalon/sequential_t1_task48_fft + resize_imgs_with_padding: + - 512 + - 512 + rtc_config: null + scheduler_decay_lr: 2.5e-06 + scheduler_decay_steps: 30000 + scheduler_warmup_steps: 1000 + self_attn_every_n_layers: 2 + tags: null + tokenizer_max_length: 48 + train_expert_only: true + train_state_proj: true + type: smolvla + use_amp: false + use_cache: true + use_delta_joint_actions_aloha: false + use_peft: false + vlm_model_name: HuggingFaceTB/SmolVLM2-500M-Video-Instruct +rabc_epsilon: + value: 1e-06 +rabc_head_mode: + value: sparse +rabc_kappa: + value: 0.01 +rabc_progress_path: + value: null +resume: + value: false +save_checkpoint: + value: true +save_freq: + value: 500 +scheduler: + value: + decay_lr: 2.5e-06 + num_decay_steps: 30000 + num_warmup_steps: 1000 + peak_lr: 2e-05 + type: cosine_decay_with_warmup +seed: + value: 42 +steps: + value: 3000 +tolerance_s: + value: 0.0001 +use_policy_training_preset: + value: true +use_rabc: + value: false +wandb: + value: + add_tags: true + disable_artifact: true + enable: true + entity: null + mode: null + notes: null + project: lerobot-smolvla + run_id: null diff --git a/wandb/run-20260408_061152-ooipb6on/files/output.log b/wandb/run-20260408_061152-ooipb6on/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..4fbbdc59a9ed729ccf43573cb371d5af0b474925 --- /dev/null +++ b/wandb/run-20260408_061152-ooipb6on/files/output.log @@ -0,0 +1,3058 @@ +INFO 2026-04-08 06:11:53 db_utils.py:117 Logs will be synced with wandb. +INFO 2026-04-08 06:11:53 db_utils.py:118 Track this run --> https://wandb.ai/aryashad-usc/lerobot-smolvla/runs/ooipb6on +INFO 2026-04-08 06:11:53 ot_train.py:221 Creating dataset +Fetching 4 files: 100% 4/4 [00:01<00:00, 2.86it/s].79MB/s] +Download complete: : 4.22MB [00:01, 2.76MB/s] +INFO 2026-04-08 06:11:55 eo_utils.py:108 Using video codec: libsvtav1 +Fetching 4 files: 100% 4/4 [00:00<00:00, 12131.03it/s] +Download complete: : 0.00B [00:00, ?B/s] +Fetching 3 files: 100% 3/3 [00:06<00:00, 2.03s/it]00:05<00:00, 101MB/s] +Download complete: 100% 481M/481M [00:06<00:00, 101MB/s] INFO 2026-04-08 06:12:02 ot_train.py:239 Creating policy +Loading HuggingFaceTB/SmolVLM2-500M-Video-Instruct weights ... +config.json: 3.77kB [00:00, 5.26MB/s] +`torch_dtype` is deprecated! Use `dtype` instead! +model.safetensors: 100% 2.03G/2.03G [00:04<00:00, 440MB/s] +Download complete: 100% 481M/481M [00:11<00:00, 41.5MB/s]] +Loading weights: 100% 489/489 [00:00<00:00, 2831.84it/s] +generation_config.json: 100% 136/136 [00:00<00:00, 322kB/s] +processor_config.json: 100% 67.0/67.0 [00:00<00:00, 167kB/s] +chat_template.json: 100% 430/430 [00:00<00:00, 1.09MB/s] +preprocessor_config.json: 100% 599/599 [00:00<00:00, 1.48MB/s] +tokenizer_config.json: 28.6kB [00:00, 34.7MB/s] +vocab.json: 801kB [00:00, 12.1MB/s] +merges.txt: 466kB [00:00, 10.9MB/s] +added_tokens.json: 4.74kB [00:00, 5.15MB/s] +special_tokens_map.json: 100% 868/868 [00:00<00:00, 2.21MB/s] +tokenizer.json: 3.55MB [00:00, 23.4MB/s] +Reducing the number of VLM layers to 16 ... +Loading weights from local directory +INFO 2026-04-08 06:12:20 ot_train.py:306 Creating optimizer and scheduler +INFO 2026-04-08 06:12:20 hedulers.py:105 Auto-scaling LR scheduler: num_training_steps (3000) < num_decay_steps (30000). Scaling warmup: 1000 → 100, decay: 30000 → 3000 (scale factor: 0.100) +INFO 2026-04-08 06:12:20 ot_train.py:341 Output dir: outputs/train/sequential_t1_task48_fft +INFO 2026-04-08 06:12:20 ot_train.py:343 cfg.env.task='libero_90' +INFO 2026-04-08 06:12:20 ot_train.py:344 Creating environment processors +INFO 2026-04-08 06:12:20 ot_train.py:348 cfg.steps=3000 (3K) +INFO 2026-04-08 06:12:20 ot_train.py:349 dataset.num_frames=7607 (8K) +INFO 2026-04-08 06:12:20 ot_train.py:350 dataset.num_episodes=45 +INFO 2026-04-08 06:12:20 ot_train.py:353 Effective batch size: 16 x 1 = 16 +INFO 2026-04-08 06:12:20 ot_train.py:354 num_learnable_params=99880992 (100M) +INFO 2026-04-08 06:12:20 ot_train.py:355 num_total_params=450046176 (450M) +Training: 0% 0/3000 [00:00, 'shape': [7]}, 'pixels/agentview_image': {'type': , 'shape': [360, 360, 3]}, 'pixels/robot0_eye_in_hand_image': {'type': , 'shape': [360, 360, 3]}, 'robot_state/eef/pos': {'type': , 'shape': [3]}, 'robot_state/eef/quat': {'type': , 'shape': [4]}, 'robot_state/eef/mat': {'type': , 'shape': [3, 3]}, 'robot_state/gripper/qpos': {'type': , 'shape': [2]}, 'robot_state/gripper/qvel': {'type': , 'shape': [2]}, 'robot_state/joints/pos': {'type': , 'shape': [7]}, 'robot_state/joints/vel': {'type': , 'shape': [7]}}, 'features_map': {'action': 'action', 'robot_state/eef/pos': 'observation.state.eef_pos', 'robot_state/eef/quat': 'observation.state.eef_quat', 'robot_state/eef/mat': 'observation.state.eef_mat', 'robot_state/gripper/qpos': 'observation.state.gripper_qpos', 'robot_state/gripper/qvel': 'observation.state.gripper_qvel', 'robot_state/joints/pos': 'observation.state.joint_pos', 'robot_state/joints/vel': 'observation.state.joint_vel', 'pixels/agentview_image': 'observation.images.image', 'pixels/robot0_eye_in_hand_image': 'observation.images.image2'}, 'max_parallel_tasks': 1, 'disable_env_checker': True, 'task_ids': [48], 'episode_length': None, 'obs_type': 'pixels_agent_pos', 'render_mode': 'rgb_array', 'camera_name': 'agentview_image,robot0_eye_in_hand_image', 'init_states': True, 'camera_name_mapping': None, 'observation_height': 360, 'observation_width': 360, 'control_mode': 'relative'}, 'policy': {'type': 'smolvla', 'n_obs_steps': 1, 'input_features': None, 'output_features': None, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': True, 'repo_id': 'ardalon/sequential_t1_task48_fft', 'private': None, 'tags': None, 'license': None, 'pretrained_path': '/content/smolvla_base', 'chunk_size': 50, 'n_action_steps': 50, 'normalization_mapping': {'VISUAL': , 'STATE': , 'ACTION': }, 'max_state_dim': 32, 'max_action_dim': 32, 'resize_imgs_with_padding': [512, 512], 'empty_cameras': 0, 'adapt_to_pi_aloha': False, 'use_delta_joint_actions_aloha': False, 'tokenizer_max_length': 48, 'num_steps': 10, 'use_cache': True, 'freeze_vision_encoder': True, 'train_expert_only': True, 'train_state_proj': True, 'optimizer_lr': 2e-05, 'optimizer_betas': [0.9, 0.95], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 0.01, 'optimizer_grad_clip_norm': 10.0, 'scheduler_warmup_steps': 1000, 'scheduler_decay_steps': 30000, 'scheduler_decay_lr': 2.5e-06, 'vlm_model_name': 'HuggingFaceTB/SmolVLM2-500M-Video-Instruct', 'load_vlm_weights': True, 'add_image_special_tokens': False, 'attention_mode': 'cross_attn', 'prefix_length': 0, 'pad_language_to': 'max_length', 'num_expert_layers': 0, 'num_vlm_layers': 16, 'self_attn_every_n_layers': 2, 'expert_width_multiplier': 0.75, 'min_period': 0.004, 'max_period': 4.0, 'rtc_config': None, 'compile_model': False, 'compile_mode': 'max-autotune'}, 'output_dir': 'outputs/train/sequential_t1_task48_fft', 'job_name': 'sequential_t1_task48_fft', 'resume': False, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 16, 'steps': 3000, 'eval_freq': 0, 'log_freq': 1, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 500, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 2e-05, 'weight_decay': 0.01, 'grad_clip_norm': 10.0, 'betas': [0.9, 0.95], 'eps': 1e-08}, 'scheduler': {'type': 'cosine_decay_with_warmup', 'num_warmup_steps': 1000, 'num_decay_steps': 30000, 'peak_lr': 2e-05, 'decay_lr': 2.5e-06}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot-smolvla', 'entity': None, 'notes': None, 'run_id': None, 'mode': None, 'add_tags': True}, 'peft': None, 'use_rabc': False, 'rabc_progress_path': None, 'rabc_kappa': 0.01, 'rabc_epsilon': 1e-06, 'rabc_head_mode': 'sparse', 'rename_map': {}, 'checkpoint_path': None, '_wandb': {}} +2026-04-08 06:11:52,307 INFO MainThread:2408 [wandb_init.py:init():892] starting backend +2026-04-08 06:11:52,644 INFO MainThread:2408 [wandb_init.py:init():895] sending inform_init request +2026-04-08 06:11:52,653 INFO MainThread:2408 [wandb_init.py:init():903] backend started and connected +2026-04-08 06:11:52,656 INFO MainThread:2408 [wandb_init.py:init():973] updated telemetry +2026-04-08 06:11:52,657 INFO MainThread:2408 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-04-08 06:11:53,136 INFO MainThread:2408 [wandb_init.py:init():1042] starting run threads in backend +2026-04-08 06:11:53,673 INFO MainThread:2408 [wandb_run.py:_console_start():2529] atexit reg +2026-04-08 06:11:53,673 INFO MainThread:2408 [wandb_run.py:_redirect():2377] redirect: wrap_raw +2026-04-08 06:11:53,673 INFO MainThread:2408 [wandb_run.py:_redirect():2446] Wrapping output streams. +2026-04-08 06:11:53,673 INFO MainThread:2408 [wandb_run.py:_redirect():2469] Redirects installed. +2026-04-08 06:11:53,677 INFO MainThread:2408 [wandb_init.py:init():1082] run started, returning control to user process +2026-04-08 06:24:24,596 INFO wandb-AsyncioManager-main:2408 [service_client.py:_forward_responses():94] Reached EOF. +2026-04-08 06:24:24,596 INFO wandb-AsyncioManager-main:2408 [mailbox.py:close():154] Closing mailbox, abandoning 1 handles. diff --git a/wandb/run-20260408_061152-ooipb6on/run-ooipb6on.wandb b/wandb/run-20260408_061152-ooipb6on/run-ooipb6on.wandb new file mode 100644 index 0000000000000000000000000000000000000000..9eeb345307ea274c06a03c91a6f7320208187468 --- /dev/null +++ b/wandb/run-20260408_061152-ooipb6on/run-ooipb6on.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94013482cc88a54b42880c771ecb607c1c25b9692e618453f361f603a06df045 +size 4578378