diff --git a/train_bsp10/checkpoints/010000/pretrained_model/config.json b/train_bsp10/checkpoints/010000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..29162bf2c105e73be74560f4dce87a7c296ad7bf --- /dev/null +++ b/train_bsp10/checkpoints/010000/pretrained_model/config.json @@ -0,0 +1,122 @@ +{ + "type": "bspline", + "n_obs_steps": 2, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.front": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.wrist": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.top": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "dtype": null, + "use_amp": false, + "use_peft": false, + "push_to_hub": false, + "repo_id": null, + "private": null, + "tags": null, + "license": null, + "pretrained_path": null, + "pretrained_revision": null, + "horizon": 16, + "n_action_steps": 32, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "drop_n_last_frames": 0, + "vision_backbone": "resnet18", + "resize_shape": [ + 120, + 160 + ], + "crop_ratio": 1.0, + "crop_shape": null, + "crop_is_random": true, + "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": true, + "down_dims": [ + 512, + 1024, + 2048 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "gradient_checkpointing": false, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.5, + "num_inference_steps": 10, + "compile_model": false, + "compile_mode": "reduce-overhead", + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "bspline_degree": 3, + "bspline_chunk_size": 10, + "bspline_window": 64, + "bspline_max_error": 0.01, + "bspline_speed_up": 1.0, + "bspline_fit_workers": 0, + "bspline_mode": "diffusion", + "planbind_fine_spacing": 4, + "planbind_subdivision": 4, + "planbind_max_plan_age": 32, + "planbind_plan_noise": 0.02, + "planbind_replan_threshold": 0.0, + "planbind_latency": 0, + "planbind_commit": "latency", + "planbind_position_limit": 0.0, + "planbind_velocity_limit": 0.0 +} \ No newline at end of file diff --git a/train_bsp10/checkpoints/010000/pretrained_model/model.safetensors b/train_bsp10/checkpoints/010000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5823e2d6c0f6cfe800bb9a6e5a40d66c331da193 --- /dev/null +++ b/train_bsp10/checkpoints/010000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:738386db4ffc1ba9b9a037dec4a2ed8f0e647d3ffb00bf5c6d39ec65d35a0d5d +size 1170940540 diff --git a/train_bsp10/checkpoints/010000/pretrained_model/policy_postprocessor.json b/train_bsp10/checkpoints/010000/pretrained_model/policy_postprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..f2c5e6cea7850bb999ab193f96021fa0129ce4db --- /dev/null +++ b/train_bsp10/checkpoints/010000/pretrained_model/policy_postprocessor.json @@ -0,0 +1,32 @@ +{ + "name": "policy_postprocessor", + "steps": [ + { + "registry_name": "unnormalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "norm_map": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + } + }, + "state_file": "policy_postprocessor_step_0_unnormalizer_processor.safetensors" + }, + { + "registry_name": "device_processor", + "config": { + "device": "cpu", + "float_dtype": null + } + } + ] +} \ No newline at end of file diff --git a/train_bsp10/checkpoints/010000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors b/train_bsp10/checkpoints/010000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ead5732518c212199e87eb8df6f0f49f32e15a1e --- /dev/null +++ b/train_bsp10/checkpoints/010000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55336fe114f19ef78b83ccc0926a3df66a9adc11a641f9adc0caf9c439ff8576 +size 8528 diff --git a/train_bsp10/checkpoints/010000/pretrained_model/policy_preprocessor.json b/train_bsp10/checkpoints/010000/pretrained_model/policy_preprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..981e0dfcb58027d80f783d8905452fadeaea830e --- /dev/null +++ b/train_bsp10/checkpoints/010000/pretrained_model/policy_preprocessor.json @@ -0,0 +1,72 @@ +{ + "name": "policy_preprocessor", + "steps": [ + { + "registry_name": "rename_observations_processor", + "config": { + "rename_map": {} + } + }, + { + "registry_name": "to_batch_processor", + "config": {} + }, + { + "registry_name": "device_processor", + "config": { + "device": "cuda", + "float_dtype": null + } + }, + { + "registry_name": "normalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.front": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.wrist": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.top": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "norm_map": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + } + }, + "state_file": "policy_preprocessor_step_3_normalizer_processor.safetensors" + } + ] +} \ No newline at end of file diff --git a/train_bsp10/checkpoints/010000/pretrained_model/policy_preprocessor_step_3_normalizer_processor.safetensors b/train_bsp10/checkpoints/010000/pretrained_model/policy_preprocessor_step_3_normalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f99bcf47b067091c10855f81f0663ca8d0f56ab8 --- /dev/null +++ b/train_bsp10/checkpoints/010000/pretrained_model/policy_preprocessor_step_3_normalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0f6cfea44fc7cfa8fa04ca706b89c90cce83136e90c3d0286428dfa2e1968d7 +size 8536 diff --git a/train_bsp10/checkpoints/010000/pretrained_model/train_config.json b/train_bsp10/checkpoints/010000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c074a87b77cd50d8d69e41de3aad8626a547e196 --- /dev/null +++ b/train_bsp10/checkpoints/010000/pretrained_model/train_config.json @@ -0,0 +1,347 @@ +{ + "dataset": { + "repo_id": "local/nexarm_stack_bowls", + "repo_type": "dataset", + "root": "/workspace/lerobot-nexarm/outputs/datasets/nexarm_stack_bowls", + "episodes": null, + "exclude_episodes": null, + "image_transforms": { + "enable": false, + "backend": "dataloader", + "gpu_compile": true, + "gpu_chunk_size": 32, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + }, + "affine": { + "weight": 1.0, + "type": "RandomAffine", + "kwargs": { + "degrees": [ + -5.0, + 5.0 + ], + "translate": [ + 0.05, + 0.05 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec", + "return_uint8": false, + "depth_output_unit": "mm", + "streaming": false, + "streaming_episode_pool_size": null, + "streaming_sampling_strategy": "remaining", + "streaming_prefetch_episodes": 8, + "streaming_byte_budget_gb": 8.0, + "streaming_decode_threads": 2, + "streaming_decoded_queue_size": 8, + "streaming_native_http_connections": null, + "streaming_native_http_subranges": 1, + "streaming_sidecar_lock_timeout_s": 1800, + "video_decoder_cache_size": null, + "eval_split": 0.0 + }, + "env": null, + "policy": { + "type": "bspline", + "n_obs_steps": 2, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.front": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.wrist": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.top": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "dtype": null, + "use_amp": false, + "use_peft": false, + "push_to_hub": false, + "repo_id": null, + "private": null, + "tags": null, + "license": null, + "pretrained_path": null, + "pretrained_revision": null, + "horizon": 16, + "n_action_steps": 32, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "drop_n_last_frames": 0, + "vision_backbone": "resnet18", + "resize_shape": [ + 120, + 160 + ], + "crop_ratio": 1.0, + "crop_shape": null, + "crop_is_random": true, + "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": true, + "down_dims": [ + 512, + 1024, + 2048 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "gradient_checkpointing": false, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.5, + "num_inference_steps": 10, + "compile_model": false, + "compile_mode": "reduce-overhead", + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "bspline_degree": 3, + "bspline_chunk_size": 10, + "bspline_window": 64, + "bspline_max_error": 0.01, + "bspline_speed_up": 1.0, + "bspline_fit_workers": 0, + "bspline_mode": "diffusion", + "planbind_fine_spacing": 4, + "planbind_subdivision": 4, + "planbind_max_plan_age": 32, + "planbind_plan_noise": 0.02, + "planbind_replan_threshold": 0.0, + "planbind_latency": 0, + "planbind_commit": "latency", + "planbind_position_limit": 0.0, + "planbind_velocity_limit": 0.0 + }, + "reward_model": null, + "output_dir": "/workspace/lerobot-nexarm/experiments/261027-plan-bind-two-step/outputs/full/train_bsp10", + "job_name": "261027-plan-bind-bsp10", + "resume": false, + "seed": 1000, + "cudnn_deterministic": false, + "num_workers": 16, + "batch_size": 32, + "prefetch_factor": 4, + "persistent_workers": true, + "dataloader_multiprocessing_context": "spawn", + "steps": 60000, + "env_eval_freq": 0, + "log_freq": 100, + "eval_steps": 0, + "max_eval_samples": 0, + "tolerance_s": 0.0001, + "save_checkpoint": true, + "save_freq": 10000, + "checkpoint_format": "safetensors", + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "fused": null + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "parallelism": { + "dp_replicate": 1, + "dp_shard": 1, + "context_parallel": { + "ring_degree": 1, + "ulysses_degree": 1 + }, + "cfg_parallel": 1 + }, + "accelerator": { + "mixed_precision": "no", + "gradient_accumulation": { + "steps": 1 + }, + "grad_scaler": { + "init_scale": 65536.0, + "growth_factor": 2.0, + "backoff_factor": 0.5, + "growth_interval": 2000 + }, + "fsdp": { + "reshard_after_forward": true, + "wrap_modules": null, + "min_num_params": null, + "cpu_offload": false, + "ignored_modules": null + }, + "ddp": { + "find_unused_parameters": true, + "gradient_as_bucket_view": false, + "static_graph": false + }, + "compile": { + "enabled": false, + "backend": "inductor", + "mode": null, + "regional": true + }, + "activation_checkpointing": { + "mode": "none" + } + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": true, + "recording": false, + "recording_repo_id": null, + "recording_private": false + }, + "ema": { + "enable": false, + "decay": null, + "update_after_step": 0, + "inv_gamma": 1.0, + "power": 0.75, + "min_decay": 0.0, + "max_decay": 0.9999, + "use_for_eval": true + }, + "wandb": { + "enable": false, + "disable_artifact": false, + "project": "lerobot-nexarm", + "entity": null, + "notes": null, + "run_id": null, + "resume": null, + "mode": null, + "console": "wrap", + "console_multipart": false, + "console_chunk_max_seconds": 0, + "add_tags": true + }, + "peft": null, + "job": { + "target": null, + "image": "huggingface/lerobot-gpu:latest", + "timeout": "2d", + "detach": false, + "tags": [] + }, + "save_checkpoint_to_hub": false, + "sample_weighting": null, + "rename_map": {}, + "checkpoint_path": null +} \ No newline at end of file diff --git a/train_bsp10/checkpoints/010000/training_state/optimizer_param_groups.json b/train_bsp10/checkpoints/010000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..9216d23351e5a0405c794b99c8a44d345c40924c --- /dev/null +++ b/train_bsp10/checkpoints/010000/training_state/optimizer_param_groups.json @@ -0,0 +1,361 @@ +[ + { + "lr": 9.384074610206495e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": false, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310, + 311, + 312, + 313, + 314, + 315, + 316, + 317, + 318, + 319, + 320, + 321, + 322, + 323, + 324, + 325, + 326, + 327, + 328, + 329, + 330, + 331, + 332, + 333, + 334, + 335, + 336, + 337, + 338, + 339 + ] + } +] \ No newline at end of file diff --git a/train_bsp10/checkpoints/010000/training_state/optimizer_state.safetensors b/train_bsp10/checkpoints/010000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8a40e080aed9743319b9e5a467859c9d236da993 --- /dev/null +++ b/train_bsp10/checkpoints/010000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45ae3b525bedbbf87eab1d54b8612c959c097737f0bd00c7d23767a0aee49dfb +size 2341614992 diff --git a/train_bsp10/checkpoints/010000/training_state/rng_state.safetensors b/train_bsp10/checkpoints/010000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7e1eec187815f057e44a0e7b2a49ab422c37d660 --- /dev/null +++ b/train_bsp10/checkpoints/010000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec93f63cd17946297a641be9b0591a88e7c9aff0b8a1f9fe393cb1760fde5559 +size 15712 diff --git a/train_bsp10/checkpoints/010000/training_state/scheduler_state.json b/train_bsp10/checkpoints/010000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..ad7b6ff1f429d32f95bbc23a5df28b7e7fa22f3f --- /dev/null +++ b/train_bsp10/checkpoints/010000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 10000, + "_step_count": 10001, + "_is_initial": false, + "_get_lr_called_within_step": false, + "_last_lr": [ + 9.384074610206495e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/train_bsp10/checkpoints/010000/training_state/training_step.json b/train_bsp10/checkpoints/010000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..8e3f6189fd774871d4f8596147e3d6f4b21b2474 --- /dev/null +++ b/train_bsp10/checkpoints/010000/training_state/training_step.json @@ -0,0 +1,13 @@ +{ + "step": 10000, + "dp_world_size": 1, + "batch_size": 32, + "grad_accum_steps": 1, + "mixed_precision": "no", + "parallelism": { + "dp_replicate": 1, + "dp_shard": 1, + "ring_degree": 1, + "ulysses_degree": 1 + } +} \ No newline at end of file diff --git a/train_bsp10/checkpoints/020000/pretrained_model/config.json b/train_bsp10/checkpoints/020000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..29162bf2c105e73be74560f4dce87a7c296ad7bf --- /dev/null +++ b/train_bsp10/checkpoints/020000/pretrained_model/config.json @@ -0,0 +1,122 @@ +{ + "type": "bspline", + "n_obs_steps": 2, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.front": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.wrist": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.top": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "dtype": null, + "use_amp": false, + "use_peft": false, + "push_to_hub": false, + "repo_id": null, + "private": null, + "tags": null, + "license": null, + "pretrained_path": null, + "pretrained_revision": null, + "horizon": 16, + "n_action_steps": 32, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "drop_n_last_frames": 0, + "vision_backbone": "resnet18", + "resize_shape": [ + 120, + 160 + ], + "crop_ratio": 1.0, + "crop_shape": null, + "crop_is_random": true, + "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": true, + "down_dims": [ + 512, + 1024, + 2048 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "gradient_checkpointing": false, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.5, + "num_inference_steps": 10, + "compile_model": false, + "compile_mode": "reduce-overhead", + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "bspline_degree": 3, + "bspline_chunk_size": 10, + "bspline_window": 64, + "bspline_max_error": 0.01, + "bspline_speed_up": 1.0, + "bspline_fit_workers": 0, + "bspline_mode": "diffusion", + "planbind_fine_spacing": 4, + "planbind_subdivision": 4, + "planbind_max_plan_age": 32, + "planbind_plan_noise": 0.02, + "planbind_replan_threshold": 0.0, + "planbind_latency": 0, + "planbind_commit": "latency", + "planbind_position_limit": 0.0, + "planbind_velocity_limit": 0.0 +} \ No newline at end of file diff --git a/train_bsp10/checkpoints/020000/pretrained_model/model.safetensors b/train_bsp10/checkpoints/020000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9941832e875dfe80482b445e0fc7490e82f42ad1 --- /dev/null +++ b/train_bsp10/checkpoints/020000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea84c51457dbf7176863003dd7ea154e1d63e3b15a8492230883836b24508146 +size 1170940540 diff --git a/train_bsp10/checkpoints/020000/pretrained_model/policy_postprocessor.json b/train_bsp10/checkpoints/020000/pretrained_model/policy_postprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..f2c5e6cea7850bb999ab193f96021fa0129ce4db --- /dev/null +++ b/train_bsp10/checkpoints/020000/pretrained_model/policy_postprocessor.json @@ -0,0 +1,32 @@ +{ + "name": "policy_postprocessor", + "steps": [ + { + "registry_name": "unnormalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "norm_map": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + } + }, + "state_file": "policy_postprocessor_step_0_unnormalizer_processor.safetensors" + }, + { + "registry_name": "device_processor", + "config": { + "device": "cpu", + "float_dtype": null + } + } + ] +} \ No newline at end of file diff --git a/train_bsp10/checkpoints/020000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors b/train_bsp10/checkpoints/020000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ead5732518c212199e87eb8df6f0f49f32e15a1e --- /dev/null +++ b/train_bsp10/checkpoints/020000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55336fe114f19ef78b83ccc0926a3df66a9adc11a641f9adc0caf9c439ff8576 +size 8528 diff --git a/train_bsp10/checkpoints/020000/pretrained_model/policy_preprocessor.json b/train_bsp10/checkpoints/020000/pretrained_model/policy_preprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..981e0dfcb58027d80f783d8905452fadeaea830e --- /dev/null +++ b/train_bsp10/checkpoints/020000/pretrained_model/policy_preprocessor.json @@ -0,0 +1,72 @@ +{ + "name": "policy_preprocessor", + "steps": [ + { + "registry_name": "rename_observations_processor", + "config": { + "rename_map": {} + } + }, + { + "registry_name": "to_batch_processor", + "config": {} + }, + { + "registry_name": "device_processor", + "config": { + "device": "cuda", + "float_dtype": null + } + }, + { + "registry_name": "normalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.front": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.wrist": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.top": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "norm_map": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + } + }, + "state_file": "policy_preprocessor_step_3_normalizer_processor.safetensors" + } + ] +} \ No newline at end of file diff --git a/train_bsp10/checkpoints/020000/pretrained_model/policy_preprocessor_step_3_normalizer_processor.safetensors b/train_bsp10/checkpoints/020000/pretrained_model/policy_preprocessor_step_3_normalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f99bcf47b067091c10855f81f0663ca8d0f56ab8 --- /dev/null +++ b/train_bsp10/checkpoints/020000/pretrained_model/policy_preprocessor_step_3_normalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0f6cfea44fc7cfa8fa04ca706b89c90cce83136e90c3d0286428dfa2e1968d7 +size 8536 diff --git a/train_bsp10/checkpoints/020000/pretrained_model/train_config.json b/train_bsp10/checkpoints/020000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c074a87b77cd50d8d69e41de3aad8626a547e196 --- /dev/null +++ b/train_bsp10/checkpoints/020000/pretrained_model/train_config.json @@ -0,0 +1,347 @@ +{ + "dataset": { + "repo_id": "local/nexarm_stack_bowls", + "repo_type": "dataset", + "root": "/workspace/lerobot-nexarm/outputs/datasets/nexarm_stack_bowls", + "episodes": null, + "exclude_episodes": null, + "image_transforms": { + "enable": false, + "backend": "dataloader", + "gpu_compile": true, + "gpu_chunk_size": 32, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + }, + "affine": { + "weight": 1.0, + "type": "RandomAffine", + "kwargs": { + "degrees": [ + -5.0, + 5.0 + ], + "translate": [ + 0.05, + 0.05 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec", + "return_uint8": false, + "depth_output_unit": "mm", + "streaming": false, + "streaming_episode_pool_size": null, + "streaming_sampling_strategy": "remaining", + "streaming_prefetch_episodes": 8, + "streaming_byte_budget_gb": 8.0, + "streaming_decode_threads": 2, + "streaming_decoded_queue_size": 8, + "streaming_native_http_connections": null, + "streaming_native_http_subranges": 1, + "streaming_sidecar_lock_timeout_s": 1800, + "video_decoder_cache_size": null, + "eval_split": 0.0 + }, + "env": null, + "policy": { + "type": "bspline", + "n_obs_steps": 2, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.front": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.wrist": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.top": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "dtype": null, + "use_amp": false, + "use_peft": false, + "push_to_hub": false, + "repo_id": null, + "private": null, + "tags": null, + "license": null, + "pretrained_path": null, + "pretrained_revision": null, + "horizon": 16, + "n_action_steps": 32, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "drop_n_last_frames": 0, + "vision_backbone": "resnet18", + "resize_shape": [ + 120, + 160 + ], + "crop_ratio": 1.0, + "crop_shape": null, + "crop_is_random": true, + "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": true, + "down_dims": [ + 512, + 1024, + 2048 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "gradient_checkpointing": false, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.5, + "num_inference_steps": 10, + "compile_model": false, + "compile_mode": "reduce-overhead", + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "bspline_degree": 3, + "bspline_chunk_size": 10, + "bspline_window": 64, + "bspline_max_error": 0.01, + "bspline_speed_up": 1.0, + "bspline_fit_workers": 0, + "bspline_mode": "diffusion", + "planbind_fine_spacing": 4, + "planbind_subdivision": 4, + "planbind_max_plan_age": 32, + "planbind_plan_noise": 0.02, + "planbind_replan_threshold": 0.0, + "planbind_latency": 0, + "planbind_commit": "latency", + "planbind_position_limit": 0.0, + "planbind_velocity_limit": 0.0 + }, + "reward_model": null, + "output_dir": "/workspace/lerobot-nexarm/experiments/261027-plan-bind-two-step/outputs/full/train_bsp10", + "job_name": "261027-plan-bind-bsp10", + "resume": false, + "seed": 1000, + "cudnn_deterministic": false, + "num_workers": 16, + "batch_size": 32, + "prefetch_factor": 4, + "persistent_workers": true, + "dataloader_multiprocessing_context": "spawn", + "steps": 60000, + "env_eval_freq": 0, + "log_freq": 100, + "eval_steps": 0, + "max_eval_samples": 0, + "tolerance_s": 0.0001, + "save_checkpoint": true, + "save_freq": 10000, + "checkpoint_format": "safetensors", + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "fused": null + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "parallelism": { + "dp_replicate": 1, + "dp_shard": 1, + "context_parallel": { + "ring_degree": 1, + "ulysses_degree": 1 + }, + "cfg_parallel": 1 + }, + "accelerator": { + "mixed_precision": "no", + "gradient_accumulation": { + "steps": 1 + }, + "grad_scaler": { + "init_scale": 65536.0, + "growth_factor": 2.0, + "backoff_factor": 0.5, + "growth_interval": 2000 + }, + "fsdp": { + "reshard_after_forward": true, + "wrap_modules": null, + "min_num_params": null, + "cpu_offload": false, + "ignored_modules": null + }, + "ddp": { + "find_unused_parameters": true, + "gradient_as_bucket_view": false, + "static_graph": false + }, + "compile": { + "enabled": false, + "backend": "inductor", + "mode": null, + "regional": true + }, + "activation_checkpointing": { + "mode": "none" + } + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": true, + "recording": false, + "recording_repo_id": null, + "recording_private": false + }, + "ema": { + "enable": false, + "decay": null, + "update_after_step": 0, + "inv_gamma": 1.0, + "power": 0.75, + "min_decay": 0.0, + "max_decay": 0.9999, + "use_for_eval": true + }, + "wandb": { + "enable": false, + "disable_artifact": false, + "project": "lerobot-nexarm", + "entity": null, + "notes": null, + "run_id": null, + "resume": null, + "mode": null, + "console": "wrap", + "console_multipart": false, + "console_chunk_max_seconds": 0, + "add_tags": true + }, + "peft": null, + "job": { + "target": null, + "image": "huggingface/lerobot-gpu:latest", + "timeout": "2d", + "detach": false, + "tags": [] + }, + "save_checkpoint_to_hub": false, + "sample_weighting": null, + "rename_map": {}, + "checkpoint_path": null +} \ No newline at end of file diff --git a/train_bsp10/checkpoints/020000/training_state/optimizer_param_groups.json b/train_bsp10/checkpoints/020000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..640aa9d5a99b7ffceceb0003deafd91f577ca5de --- /dev/null +++ b/train_bsp10/checkpoints/020000/training_state/optimizer_param_groups.json @@ -0,0 +1,361 @@ +[ + { + "lr": 7.57581893473448e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": false, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310, + 311, + 312, + 313, + 314, + 315, + 316, + 317, + 318, + 319, + 320, + 321, + 322, + 323, + 324, + 325, + 326, + 327, + 328, + 329, + 330, + 331, + 332, + 333, + 334, + 335, + 336, + 337, + 338, + 339 + ] + } +] \ No newline at end of file diff --git a/train_bsp10/checkpoints/020000/training_state/optimizer_state.safetensors b/train_bsp10/checkpoints/020000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..858b28dce62f0d8b0b5f59a7df8172edc4c28938 --- /dev/null +++ b/train_bsp10/checkpoints/020000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a057607b24eced86b08b15ee733d8c548ca3ab2255f1f3486393e30eb404742f +size 2341614992 diff --git a/train_bsp10/checkpoints/020000/training_state/rng_state.safetensors b/train_bsp10/checkpoints/020000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c4e487196d48f578f3271d1d04a5fc779a5ebbb9 --- /dev/null +++ b/train_bsp10/checkpoints/020000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e21b4125a095d344fb18538f45745cb73ff81a6cd3d96f3b79d084a81024137 +size 15712 diff --git a/train_bsp10/checkpoints/020000/training_state/scheduler_state.json b/train_bsp10/checkpoints/020000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..703da730043074dddcd2d1f88fdea38f195ce3ee --- /dev/null +++ b/train_bsp10/checkpoints/020000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 20000, + "_step_count": 20001, + "_is_initial": false, + "_get_lr_called_within_step": false, + "_last_lr": [ + 7.57581893473448e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/train_bsp10/checkpoints/020000/training_state/training_step.json b/train_bsp10/checkpoints/020000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..79cc396869d8aa4a40c8c0369eb032edf5d69e78 --- /dev/null +++ b/train_bsp10/checkpoints/020000/training_state/training_step.json @@ -0,0 +1,13 @@ +{ + "step": 20000, + "dp_world_size": 1, + "batch_size": 32, + "grad_accum_steps": 1, + "mixed_precision": "no", + "parallelism": { + "dp_replicate": 1, + "dp_shard": 1, + "ring_degree": 1, + "ulysses_degree": 1 + } +} \ No newline at end of file diff --git a/train_bsp10/checkpoints/030000/pretrained_model/config.json b/train_bsp10/checkpoints/030000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..29162bf2c105e73be74560f4dce87a7c296ad7bf --- /dev/null +++ b/train_bsp10/checkpoints/030000/pretrained_model/config.json @@ -0,0 +1,122 @@ +{ + "type": "bspline", + "n_obs_steps": 2, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.front": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.wrist": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.top": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "dtype": null, + "use_amp": false, + "use_peft": false, + "push_to_hub": false, + "repo_id": null, + "private": null, + "tags": null, + "license": null, + "pretrained_path": null, + "pretrained_revision": null, + "horizon": 16, + "n_action_steps": 32, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "drop_n_last_frames": 0, + "vision_backbone": "resnet18", + "resize_shape": [ + 120, + 160 + ], + "crop_ratio": 1.0, + "crop_shape": null, + "crop_is_random": true, + "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": true, + "down_dims": [ + 512, + 1024, + 2048 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "gradient_checkpointing": false, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.5, + "num_inference_steps": 10, + "compile_model": false, + "compile_mode": "reduce-overhead", + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "bspline_degree": 3, + "bspline_chunk_size": 10, + "bspline_window": 64, + "bspline_max_error": 0.01, + "bspline_speed_up": 1.0, + "bspline_fit_workers": 0, + "bspline_mode": "diffusion", + "planbind_fine_spacing": 4, + "planbind_subdivision": 4, + "planbind_max_plan_age": 32, + "planbind_plan_noise": 0.02, + "planbind_replan_threshold": 0.0, + "planbind_latency": 0, + "planbind_commit": "latency", + "planbind_position_limit": 0.0, + "planbind_velocity_limit": 0.0 +} \ No newline at end of file diff --git a/train_bsp10/checkpoints/030000/pretrained_model/model.safetensors b/train_bsp10/checkpoints/030000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..447faccb83a7a3534c6de9f82f317e7fe2fef575 --- /dev/null +++ b/train_bsp10/checkpoints/030000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f1509164ff045a4e1fa7d146a8d4b2402cf7c66c64410653e3ae438b036d49c +size 1170940540 diff --git a/train_bsp10/checkpoints/030000/pretrained_model/policy_postprocessor.json b/train_bsp10/checkpoints/030000/pretrained_model/policy_postprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..f2c5e6cea7850bb999ab193f96021fa0129ce4db --- /dev/null +++ b/train_bsp10/checkpoints/030000/pretrained_model/policy_postprocessor.json @@ -0,0 +1,32 @@ +{ + "name": "policy_postprocessor", + "steps": [ + { + "registry_name": "unnormalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "norm_map": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + } + }, + "state_file": "policy_postprocessor_step_0_unnormalizer_processor.safetensors" + }, + { + "registry_name": "device_processor", + "config": { + "device": "cpu", + "float_dtype": null + } + } + ] +} \ No newline at end of file diff --git a/train_bsp10/checkpoints/030000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors b/train_bsp10/checkpoints/030000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ead5732518c212199e87eb8df6f0f49f32e15a1e --- /dev/null +++ b/train_bsp10/checkpoints/030000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55336fe114f19ef78b83ccc0926a3df66a9adc11a641f9adc0caf9c439ff8576 +size 8528 diff --git a/train_bsp10/checkpoints/030000/pretrained_model/policy_preprocessor.json b/train_bsp10/checkpoints/030000/pretrained_model/policy_preprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..981e0dfcb58027d80f783d8905452fadeaea830e --- /dev/null +++ b/train_bsp10/checkpoints/030000/pretrained_model/policy_preprocessor.json @@ -0,0 +1,72 @@ +{ + "name": "policy_preprocessor", + "steps": [ + { + "registry_name": "rename_observations_processor", + "config": { + "rename_map": {} + } + }, + { + "registry_name": "to_batch_processor", + "config": {} + }, + { + "registry_name": "device_processor", + "config": { + "device": "cuda", + "float_dtype": null + } + }, + { + "registry_name": "normalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.front": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.wrist": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.top": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "norm_map": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + } + }, + "state_file": "policy_preprocessor_step_3_normalizer_processor.safetensors" + } + ] +} \ No newline at end of file diff --git a/train_bsp10/checkpoints/030000/pretrained_model/policy_preprocessor_step_3_normalizer_processor.safetensors b/train_bsp10/checkpoints/030000/pretrained_model/policy_preprocessor_step_3_normalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f99bcf47b067091c10855f81f0663ca8d0f56ab8 --- /dev/null +++ b/train_bsp10/checkpoints/030000/pretrained_model/policy_preprocessor_step_3_normalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0f6cfea44fc7cfa8fa04ca706b89c90cce83136e90c3d0286428dfa2e1968d7 +size 8536 diff --git a/train_bsp10/checkpoints/030000/pretrained_model/train_config.json b/train_bsp10/checkpoints/030000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c074a87b77cd50d8d69e41de3aad8626a547e196 --- /dev/null +++ b/train_bsp10/checkpoints/030000/pretrained_model/train_config.json @@ -0,0 +1,347 @@ +{ + "dataset": { + "repo_id": "local/nexarm_stack_bowls", + "repo_type": "dataset", + "root": "/workspace/lerobot-nexarm/outputs/datasets/nexarm_stack_bowls", + "episodes": null, + "exclude_episodes": null, + "image_transforms": { + "enable": false, + "backend": "dataloader", + "gpu_compile": true, + "gpu_chunk_size": 32, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + }, + "affine": { + "weight": 1.0, + "type": "RandomAffine", + "kwargs": { + "degrees": [ + -5.0, + 5.0 + ], + "translate": [ + 0.05, + 0.05 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec", + "return_uint8": false, + "depth_output_unit": "mm", + "streaming": false, + "streaming_episode_pool_size": null, + "streaming_sampling_strategy": "remaining", + "streaming_prefetch_episodes": 8, + "streaming_byte_budget_gb": 8.0, + "streaming_decode_threads": 2, + "streaming_decoded_queue_size": 8, + "streaming_native_http_connections": null, + "streaming_native_http_subranges": 1, + "streaming_sidecar_lock_timeout_s": 1800, + "video_decoder_cache_size": null, + "eval_split": 0.0 + }, + "env": null, + "policy": { + "type": "bspline", + "n_obs_steps": 2, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.front": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.wrist": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.top": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "dtype": null, + "use_amp": false, + "use_peft": false, + "push_to_hub": false, + "repo_id": null, + "private": null, + "tags": null, + "license": null, + "pretrained_path": null, + "pretrained_revision": null, + "horizon": 16, + "n_action_steps": 32, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "drop_n_last_frames": 0, + "vision_backbone": "resnet18", + "resize_shape": [ + 120, + 160 + ], + "crop_ratio": 1.0, + "crop_shape": null, + "crop_is_random": true, + "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": true, + "down_dims": [ + 512, + 1024, + 2048 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "gradient_checkpointing": false, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.5, + "num_inference_steps": 10, + "compile_model": false, + "compile_mode": "reduce-overhead", + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "bspline_degree": 3, + "bspline_chunk_size": 10, + "bspline_window": 64, + "bspline_max_error": 0.01, + "bspline_speed_up": 1.0, + "bspline_fit_workers": 0, + "bspline_mode": "diffusion", + "planbind_fine_spacing": 4, + "planbind_subdivision": 4, + "planbind_max_plan_age": 32, + "planbind_plan_noise": 0.02, + "planbind_replan_threshold": 0.0, + "planbind_latency": 0, + "planbind_commit": "latency", + "planbind_position_limit": 0.0, + "planbind_velocity_limit": 0.0 + }, + "reward_model": null, + "output_dir": "/workspace/lerobot-nexarm/experiments/261027-plan-bind-two-step/outputs/full/train_bsp10", + "job_name": "261027-plan-bind-bsp10", + "resume": false, + "seed": 1000, + "cudnn_deterministic": false, + "num_workers": 16, + "batch_size": 32, + "prefetch_factor": 4, + "persistent_workers": true, + "dataloader_multiprocessing_context": "spawn", + "steps": 60000, + "env_eval_freq": 0, + "log_freq": 100, + "eval_steps": 0, + "max_eval_samples": 0, + "tolerance_s": 0.0001, + "save_checkpoint": true, + "save_freq": 10000, + "checkpoint_format": "safetensors", + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "fused": null + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "parallelism": { + "dp_replicate": 1, + "dp_shard": 1, + "context_parallel": { + "ring_degree": 1, + "ulysses_degree": 1 + }, + "cfg_parallel": 1 + }, + "accelerator": { + "mixed_precision": "no", + "gradient_accumulation": { + "steps": 1 + }, + "grad_scaler": { + "init_scale": 65536.0, + "growth_factor": 2.0, + "backoff_factor": 0.5, + "growth_interval": 2000 + }, + "fsdp": { + "reshard_after_forward": true, + "wrap_modules": null, + "min_num_params": null, + "cpu_offload": false, + "ignored_modules": null + }, + "ddp": { + "find_unused_parameters": true, + "gradient_as_bucket_view": false, + "static_graph": false + }, + "compile": { + "enabled": false, + "backend": "inductor", + "mode": null, + "regional": true + }, + "activation_checkpointing": { + "mode": "none" + } + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": true, + "recording": false, + "recording_repo_id": null, + "recording_private": false + }, + "ema": { + "enable": false, + "decay": null, + "update_after_step": 0, + "inv_gamma": 1.0, + "power": 0.75, + "min_decay": 0.0, + "max_decay": 0.9999, + "use_for_eval": true + }, + "wandb": { + "enable": false, + "disable_artifact": false, + "project": "lerobot-nexarm", + "entity": null, + "notes": null, + "run_id": null, + "resume": null, + "mode": null, + "console": "wrap", + "console_multipart": false, + "console_chunk_max_seconds": 0, + "add_tags": true + }, + "peft": null, + "job": { + "target": null, + "image": "huggingface/lerobot-gpu:latest", + "timeout": "2d", + "detach": false, + "tags": [] + }, + "save_checkpoint_to_hub": false, + "sample_weighting": null, + "rename_map": {}, + "checkpoint_path": null +} \ No newline at end of file diff --git a/train_bsp10/checkpoints/030000/training_state/optimizer_param_groups.json b/train_bsp10/checkpoints/030000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..1e65d4fa7397591ca9a71c24645f4817b6492be7 --- /dev/null +++ b/train_bsp10/checkpoints/030000/training_state/optimizer_param_groups.json @@ -0,0 +1,361 @@ +[ + { + "lr": 5.0659979290537954e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": false, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310, + 311, + 312, + 313, + 314, + 315, + 316, + 317, + 318, + 319, + 320, + 321, + 322, + 323, + 324, + 325, + 326, + 327, + 328, + 329, + 330, + 331, + 332, + 333, + 334, + 335, + 336, + 337, + 338, + 339 + ] + } +] \ No newline at end of file diff --git a/train_bsp10/checkpoints/030000/training_state/optimizer_state.safetensors b/train_bsp10/checkpoints/030000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e5c0b68e696eaee9e80433dcb3508ac50702f9d0 --- /dev/null +++ b/train_bsp10/checkpoints/030000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:edffbb21df7692486d242dafb85d5b156bbf606791ec7edf44af3730908e85b7 +size 2341614992 diff --git a/train_bsp10/checkpoints/030000/training_state/rng_state.safetensors b/train_bsp10/checkpoints/030000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5197db2c79c74ecbde1d97d69647589a727da188 --- /dev/null +++ b/train_bsp10/checkpoints/030000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0fa2590dae55a570fa46905c969d3fbc69218ae12820ffbc9d3cad8a6f85a84b +size 15712 diff --git a/train_bsp10/checkpoints/030000/training_state/scheduler_state.json b/train_bsp10/checkpoints/030000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..18610eb37081452a6ecc2e858583500e0ad76d58 --- /dev/null +++ b/train_bsp10/checkpoints/030000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 30000, + "_step_count": 30001, + "_is_initial": false, + "_get_lr_called_within_step": false, + "_last_lr": [ + 5.0659979290537954e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/train_bsp10/checkpoints/030000/training_state/training_step.json b/train_bsp10/checkpoints/030000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..b90b0a337eea7f2c3dc1f6f7c3954155e67e35ff --- /dev/null +++ b/train_bsp10/checkpoints/030000/training_state/training_step.json @@ -0,0 +1,13 @@ +{ + "step": 30000, + "dp_world_size": 1, + "batch_size": 32, + "grad_accum_steps": 1, + "mixed_precision": "no", + "parallelism": { + "dp_replicate": 1, + "dp_shard": 1, + "ring_degree": 1, + "ulysses_degree": 1 + } +} \ No newline at end of file diff --git a/train_bsp10/checkpoints/040000/pretrained_model/config.json b/train_bsp10/checkpoints/040000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..29162bf2c105e73be74560f4dce87a7c296ad7bf --- /dev/null +++ b/train_bsp10/checkpoints/040000/pretrained_model/config.json @@ -0,0 +1,122 @@ +{ + "type": "bspline", + "n_obs_steps": 2, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.front": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.wrist": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.top": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "dtype": null, + "use_amp": false, + "use_peft": false, + "push_to_hub": false, + "repo_id": null, + "private": null, + "tags": null, + "license": null, + "pretrained_path": null, + "pretrained_revision": null, + "horizon": 16, + "n_action_steps": 32, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "drop_n_last_frames": 0, + "vision_backbone": "resnet18", + "resize_shape": [ + 120, + 160 + ], + "crop_ratio": 1.0, + "crop_shape": null, + "crop_is_random": true, + "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": true, + "down_dims": [ + 512, + 1024, + 2048 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "gradient_checkpointing": false, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.5, + "num_inference_steps": 10, + "compile_model": false, + "compile_mode": "reduce-overhead", + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "bspline_degree": 3, + "bspline_chunk_size": 10, + "bspline_window": 64, + "bspline_max_error": 0.01, + "bspline_speed_up": 1.0, + "bspline_fit_workers": 0, + "bspline_mode": "diffusion", + "planbind_fine_spacing": 4, + "planbind_subdivision": 4, + "planbind_max_plan_age": 32, + "planbind_plan_noise": 0.02, + "planbind_replan_threshold": 0.0, + "planbind_latency": 0, + "planbind_commit": "latency", + "planbind_position_limit": 0.0, + "planbind_velocity_limit": 0.0 +} \ No newline at end of file diff --git a/train_bsp10/checkpoints/040000/pretrained_model/model.safetensors b/train_bsp10/checkpoints/040000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ab0ba27172ae1f1331e5402af8be8892ea0b8697 --- /dev/null +++ b/train_bsp10/checkpoints/040000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:044bf12183dd7fd90effa05bb0294f9c10e0cd2a429aa86ca1e2aef1487fb7be +size 1170940540 diff --git a/train_bsp10/checkpoints/040000/pretrained_model/policy_postprocessor.json b/train_bsp10/checkpoints/040000/pretrained_model/policy_postprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..f2c5e6cea7850bb999ab193f96021fa0129ce4db --- /dev/null +++ b/train_bsp10/checkpoints/040000/pretrained_model/policy_postprocessor.json @@ -0,0 +1,32 @@ +{ + "name": "policy_postprocessor", + "steps": [ + { + "registry_name": "unnormalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "norm_map": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + } + }, + "state_file": "policy_postprocessor_step_0_unnormalizer_processor.safetensors" + }, + { + "registry_name": "device_processor", + "config": { + "device": "cpu", + "float_dtype": null + } + } + ] +} \ No newline at end of file diff --git a/train_bsp10/checkpoints/040000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors b/train_bsp10/checkpoints/040000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ead5732518c212199e87eb8df6f0f49f32e15a1e --- /dev/null +++ b/train_bsp10/checkpoints/040000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55336fe114f19ef78b83ccc0926a3df66a9adc11a641f9adc0caf9c439ff8576 +size 8528 diff --git a/train_bsp10/checkpoints/040000/pretrained_model/policy_preprocessor.json b/train_bsp10/checkpoints/040000/pretrained_model/policy_preprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..981e0dfcb58027d80f783d8905452fadeaea830e --- /dev/null +++ b/train_bsp10/checkpoints/040000/pretrained_model/policy_preprocessor.json @@ -0,0 +1,72 @@ +{ + "name": "policy_preprocessor", + "steps": [ + { + "registry_name": "rename_observations_processor", + "config": { + "rename_map": {} + } + }, + { + "registry_name": "to_batch_processor", + "config": {} + }, + { + "registry_name": "device_processor", + "config": { + "device": "cuda", + "float_dtype": null + } + }, + { + "registry_name": "normalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.front": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.wrist": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.top": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "norm_map": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + } + }, + "state_file": "policy_preprocessor_step_3_normalizer_processor.safetensors" + } + ] +} \ No newline at end of file diff --git a/train_bsp10/checkpoints/040000/pretrained_model/policy_preprocessor_step_3_normalizer_processor.safetensors b/train_bsp10/checkpoints/040000/pretrained_model/policy_preprocessor_step_3_normalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f99bcf47b067091c10855f81f0663ca8d0f56ab8 --- /dev/null +++ b/train_bsp10/checkpoints/040000/pretrained_model/policy_preprocessor_step_3_normalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0f6cfea44fc7cfa8fa04ca706b89c90cce83136e90c3d0286428dfa2e1968d7 +size 8536 diff --git a/train_bsp10/checkpoints/040000/pretrained_model/train_config.json b/train_bsp10/checkpoints/040000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c074a87b77cd50d8d69e41de3aad8626a547e196 --- /dev/null +++ b/train_bsp10/checkpoints/040000/pretrained_model/train_config.json @@ -0,0 +1,347 @@ +{ + "dataset": { + "repo_id": "local/nexarm_stack_bowls", + "repo_type": "dataset", + "root": "/workspace/lerobot-nexarm/outputs/datasets/nexarm_stack_bowls", + "episodes": null, + "exclude_episodes": null, + "image_transforms": { + "enable": false, + "backend": "dataloader", + "gpu_compile": true, + "gpu_chunk_size": 32, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + }, + "affine": { + "weight": 1.0, + "type": "RandomAffine", + "kwargs": { + "degrees": [ + -5.0, + 5.0 + ], + "translate": [ + 0.05, + 0.05 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec", + "return_uint8": false, + "depth_output_unit": "mm", + "streaming": false, + "streaming_episode_pool_size": null, + "streaming_sampling_strategy": "remaining", + "streaming_prefetch_episodes": 8, + "streaming_byte_budget_gb": 8.0, + "streaming_decode_threads": 2, + "streaming_decoded_queue_size": 8, + "streaming_native_http_connections": null, + "streaming_native_http_subranges": 1, + "streaming_sidecar_lock_timeout_s": 1800, + "video_decoder_cache_size": null, + "eval_split": 0.0 + }, + "env": null, + "policy": { + "type": "bspline", + "n_obs_steps": 2, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.front": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.wrist": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.top": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "dtype": null, + "use_amp": false, + "use_peft": false, + "push_to_hub": false, + "repo_id": null, + "private": null, + "tags": null, + "license": null, + "pretrained_path": null, + "pretrained_revision": null, + "horizon": 16, + "n_action_steps": 32, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "drop_n_last_frames": 0, + "vision_backbone": "resnet18", + "resize_shape": [ + 120, + 160 + ], + "crop_ratio": 1.0, + "crop_shape": null, + "crop_is_random": true, + "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": true, + "down_dims": [ + 512, + 1024, + 2048 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "gradient_checkpointing": false, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.5, + "num_inference_steps": 10, + "compile_model": false, + "compile_mode": "reduce-overhead", + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "bspline_degree": 3, + "bspline_chunk_size": 10, + "bspline_window": 64, + "bspline_max_error": 0.01, + "bspline_speed_up": 1.0, + "bspline_fit_workers": 0, + "bspline_mode": "diffusion", + "planbind_fine_spacing": 4, + "planbind_subdivision": 4, + "planbind_max_plan_age": 32, + "planbind_plan_noise": 0.02, + "planbind_replan_threshold": 0.0, + "planbind_latency": 0, + "planbind_commit": "latency", + "planbind_position_limit": 0.0, + "planbind_velocity_limit": 0.0 + }, + "reward_model": null, + "output_dir": "/workspace/lerobot-nexarm/experiments/261027-plan-bind-two-step/outputs/full/train_bsp10", + "job_name": "261027-plan-bind-bsp10", + "resume": false, + "seed": 1000, + "cudnn_deterministic": false, + "num_workers": 16, + "batch_size": 32, + "prefetch_factor": 4, + "persistent_workers": true, + "dataloader_multiprocessing_context": "spawn", + "steps": 60000, + "env_eval_freq": 0, + "log_freq": 100, + "eval_steps": 0, + "max_eval_samples": 0, + "tolerance_s": 0.0001, + "save_checkpoint": true, + "save_freq": 10000, + "checkpoint_format": "safetensors", + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "fused": null + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "parallelism": { + "dp_replicate": 1, + "dp_shard": 1, + "context_parallel": { + "ring_degree": 1, + "ulysses_degree": 1 + }, + "cfg_parallel": 1 + }, + "accelerator": { + "mixed_precision": "no", + "gradient_accumulation": { + "steps": 1 + }, + "grad_scaler": { + "init_scale": 65536.0, + "growth_factor": 2.0, + "backoff_factor": 0.5, + "growth_interval": 2000 + }, + "fsdp": { + "reshard_after_forward": true, + "wrap_modules": null, + "min_num_params": null, + "cpu_offload": false, + "ignored_modules": null + }, + "ddp": { + "find_unused_parameters": true, + "gradient_as_bucket_view": false, + "static_graph": false + }, + "compile": { + "enabled": false, + "backend": "inductor", + "mode": null, + "regional": true + }, + "activation_checkpointing": { + "mode": "none" + } + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": true, + "recording": false, + "recording_repo_id": null, + "recording_private": false + }, + "ema": { + "enable": false, + "decay": null, + "update_after_step": 0, + "inv_gamma": 1.0, + "power": 0.75, + "min_decay": 0.0, + "max_decay": 0.9999, + "use_for_eval": true + }, + "wandb": { + "enable": false, + "disable_artifact": false, + "project": "lerobot-nexarm", + "entity": null, + "notes": null, + "run_id": null, + "resume": null, + "mode": null, + "console": "wrap", + "console_multipart": false, + "console_chunk_max_seconds": 0, + "add_tags": true + }, + "peft": null, + "job": { + "target": null, + "image": "huggingface/lerobot-gpu:latest", + "timeout": "2d", + "detach": false, + "tags": [] + }, + "save_checkpoint_to_hub": false, + "sample_weighting": null, + "rename_map": {}, + "checkpoint_path": null +} \ No newline at end of file diff --git a/train_bsp10/checkpoints/040000/training_state/optimizer_param_groups.json b/train_bsp10/checkpoints/040000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..ab49b07a4bc14a7a219cfcb6611ba784bef4323f --- /dev/null +++ b/train_bsp10/checkpoints/040000/training_state/optimizer_param_groups.json @@ -0,0 +1,361 @@ +[ + { + "lr": 2.5382013357782893e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": false, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310, + 311, + 312, + 313, + 314, + 315, + 316, + 317, + 318, + 319, + 320, + 321, + 322, + 323, + 324, + 325, + 326, + 327, + 328, + 329, + 330, + 331, + 332, + 333, + 334, + 335, + 336, + 337, + 338, + 339 + ] + } +] \ No newline at end of file diff --git a/train_bsp10/checkpoints/040000/training_state/optimizer_state.safetensors b/train_bsp10/checkpoints/040000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c4e551344b17c3267baf3910d245ec51df7697dd --- /dev/null +++ b/train_bsp10/checkpoints/040000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a48666104970d4ec44e281e496f3a7010c389dc724f7563f1a7cfee1b37c289 +size 2341614992 diff --git a/train_bsp10/checkpoints/040000/training_state/rng_state.safetensors b/train_bsp10/checkpoints/040000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f58ef510395f2c45c1d0bad0a3e1400be123e013 --- /dev/null +++ b/train_bsp10/checkpoints/040000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46c794652ed7ffddac7d330c051b40956690034b4a9a1d7a1a47b8b231c10d48 +size 15712 diff --git a/train_bsp10/checkpoints/040000/training_state/scheduler_state.json b/train_bsp10/checkpoints/040000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..57516318c0fbc503a08ada720c8e48a64fb460ba --- /dev/null +++ b/train_bsp10/checkpoints/040000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 40000, + "_step_count": 40001, + "_is_initial": false, + "_get_lr_called_within_step": false, + "_last_lr": [ + 2.5382013357782893e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/train_bsp10/checkpoints/040000/training_state/training_step.json b/train_bsp10/checkpoints/040000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..986d2ecdfcf0680c6602a1fd26360fb009bc16fb --- /dev/null +++ b/train_bsp10/checkpoints/040000/training_state/training_step.json @@ -0,0 +1,13 @@ +{ + "step": 40000, + "dp_world_size": 1, + "batch_size": 32, + "grad_accum_steps": 1, + "mixed_precision": "no", + "parallelism": { + "dp_replicate": 1, + "dp_shard": 1, + "ring_degree": 1, + "ulysses_degree": 1 + } +} \ No newline at end of file diff --git a/train_bsp10/checkpoints/050000/pretrained_model/config.json b/train_bsp10/checkpoints/050000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..29162bf2c105e73be74560f4dce87a7c296ad7bf --- /dev/null +++ b/train_bsp10/checkpoints/050000/pretrained_model/config.json @@ -0,0 +1,122 @@ +{ + "type": "bspline", + "n_obs_steps": 2, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.front": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.wrist": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.top": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "dtype": null, + "use_amp": false, + "use_peft": false, + "push_to_hub": false, + "repo_id": null, + "private": null, + "tags": null, + "license": null, + "pretrained_path": null, + "pretrained_revision": null, + "horizon": 16, + "n_action_steps": 32, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "drop_n_last_frames": 0, + "vision_backbone": "resnet18", + "resize_shape": [ + 120, + 160 + ], + "crop_ratio": 1.0, + "crop_shape": null, + "crop_is_random": true, + "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": true, + "down_dims": [ + 512, + 1024, + 2048 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "gradient_checkpointing": false, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.5, + "num_inference_steps": 10, + "compile_model": false, + "compile_mode": "reduce-overhead", + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "bspline_degree": 3, + "bspline_chunk_size": 10, + "bspline_window": 64, + "bspline_max_error": 0.01, + "bspline_speed_up": 1.0, + "bspline_fit_workers": 0, + "bspline_mode": "diffusion", + "planbind_fine_spacing": 4, + "planbind_subdivision": 4, + "planbind_max_plan_age": 32, + "planbind_plan_noise": 0.02, + "planbind_replan_threshold": 0.0, + "planbind_latency": 0, + "planbind_commit": "latency", + "planbind_position_limit": 0.0, + "planbind_velocity_limit": 0.0 +} \ No newline at end of file diff --git a/train_bsp10/checkpoints/050000/pretrained_model/model.safetensors b/train_bsp10/checkpoints/050000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..618d70761a0242493f545c8a2e856c98f1753913 --- /dev/null +++ b/train_bsp10/checkpoints/050000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f157b83f20847df97550c23f1f747afa2489a52536d8aba43b5edc179e2ba9f6 +size 1170940540 diff --git a/train_bsp10/checkpoints/050000/pretrained_model/policy_postprocessor.json b/train_bsp10/checkpoints/050000/pretrained_model/policy_postprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..f2c5e6cea7850bb999ab193f96021fa0129ce4db --- /dev/null +++ b/train_bsp10/checkpoints/050000/pretrained_model/policy_postprocessor.json @@ -0,0 +1,32 @@ +{ + "name": "policy_postprocessor", + "steps": [ + { + "registry_name": "unnormalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "norm_map": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + } + }, + "state_file": "policy_postprocessor_step_0_unnormalizer_processor.safetensors" + }, + { + "registry_name": "device_processor", + "config": { + "device": "cpu", + "float_dtype": null + } + } + ] +} \ No newline at end of file diff --git a/train_bsp10/checkpoints/050000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors b/train_bsp10/checkpoints/050000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ead5732518c212199e87eb8df6f0f49f32e15a1e --- /dev/null +++ b/train_bsp10/checkpoints/050000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55336fe114f19ef78b83ccc0926a3df66a9adc11a641f9adc0caf9c439ff8576 +size 8528 diff --git a/train_bsp10/checkpoints/050000/pretrained_model/policy_preprocessor.json b/train_bsp10/checkpoints/050000/pretrained_model/policy_preprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..981e0dfcb58027d80f783d8905452fadeaea830e --- /dev/null +++ b/train_bsp10/checkpoints/050000/pretrained_model/policy_preprocessor.json @@ -0,0 +1,72 @@ +{ + "name": "policy_preprocessor", + "steps": [ + { + "registry_name": "rename_observations_processor", + "config": { + "rename_map": {} + } + }, + { + "registry_name": "to_batch_processor", + "config": {} + }, + { + "registry_name": "device_processor", + "config": { + "device": "cuda", + "float_dtype": null + } + }, + { + "registry_name": "normalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.front": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.wrist": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.top": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "norm_map": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + } + }, + "state_file": "policy_preprocessor_step_3_normalizer_processor.safetensors" + } + ] +} \ No newline at end of file diff --git a/train_bsp10/checkpoints/050000/pretrained_model/policy_preprocessor_step_3_normalizer_processor.safetensors b/train_bsp10/checkpoints/050000/pretrained_model/policy_preprocessor_step_3_normalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f99bcf47b067091c10855f81f0663ca8d0f56ab8 --- /dev/null +++ b/train_bsp10/checkpoints/050000/pretrained_model/policy_preprocessor_step_3_normalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0f6cfea44fc7cfa8fa04ca706b89c90cce83136e90c3d0286428dfa2e1968d7 +size 8536 diff --git a/train_bsp10/checkpoints/050000/pretrained_model/train_config.json b/train_bsp10/checkpoints/050000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c074a87b77cd50d8d69e41de3aad8626a547e196 --- /dev/null +++ b/train_bsp10/checkpoints/050000/pretrained_model/train_config.json @@ -0,0 +1,347 @@ +{ + "dataset": { + "repo_id": "local/nexarm_stack_bowls", + "repo_type": "dataset", + "root": "/workspace/lerobot-nexarm/outputs/datasets/nexarm_stack_bowls", + "episodes": null, + "exclude_episodes": null, + "image_transforms": { + "enable": false, + "backend": "dataloader", + "gpu_compile": true, + "gpu_chunk_size": 32, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + }, + "affine": { + "weight": 1.0, + "type": "RandomAffine", + "kwargs": { + "degrees": [ + -5.0, + 5.0 + ], + "translate": [ + 0.05, + 0.05 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec", + "return_uint8": false, + "depth_output_unit": "mm", + "streaming": false, + "streaming_episode_pool_size": null, + "streaming_sampling_strategy": "remaining", + "streaming_prefetch_episodes": 8, + "streaming_byte_budget_gb": 8.0, + "streaming_decode_threads": 2, + "streaming_decoded_queue_size": 8, + "streaming_native_http_connections": null, + "streaming_native_http_subranges": 1, + "streaming_sidecar_lock_timeout_s": 1800, + "video_decoder_cache_size": null, + "eval_split": 0.0 + }, + "env": null, + "policy": { + "type": "bspline", + "n_obs_steps": 2, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.front": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.wrist": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.top": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "dtype": null, + "use_amp": false, + "use_peft": false, + "push_to_hub": false, + "repo_id": null, + "private": null, + "tags": null, + "license": null, + "pretrained_path": null, + "pretrained_revision": null, + "horizon": 16, + "n_action_steps": 32, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "drop_n_last_frames": 0, + "vision_backbone": "resnet18", + "resize_shape": [ + 120, + 160 + ], + "crop_ratio": 1.0, + "crop_shape": null, + "crop_is_random": true, + "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": true, + "down_dims": [ + 512, + 1024, + 2048 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "gradient_checkpointing": false, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.5, + "num_inference_steps": 10, + "compile_model": false, + "compile_mode": "reduce-overhead", + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "bspline_degree": 3, + "bspline_chunk_size": 10, + "bspline_window": 64, + "bspline_max_error": 0.01, + "bspline_speed_up": 1.0, + "bspline_fit_workers": 0, + "bspline_mode": "diffusion", + "planbind_fine_spacing": 4, + "planbind_subdivision": 4, + "planbind_max_plan_age": 32, + "planbind_plan_noise": 0.02, + "planbind_replan_threshold": 0.0, + "planbind_latency": 0, + "planbind_commit": "latency", + "planbind_position_limit": 0.0, + "planbind_velocity_limit": 0.0 + }, + "reward_model": null, + "output_dir": "/workspace/lerobot-nexarm/experiments/261027-plan-bind-two-step/outputs/full/train_bsp10", + "job_name": "261027-plan-bind-bsp10", + "resume": false, + "seed": 1000, + "cudnn_deterministic": false, + "num_workers": 16, + "batch_size": 32, + "prefetch_factor": 4, + "persistent_workers": true, + "dataloader_multiprocessing_context": "spawn", + "steps": 60000, + "env_eval_freq": 0, + "log_freq": 100, + "eval_steps": 0, + "max_eval_samples": 0, + "tolerance_s": 0.0001, + "save_checkpoint": true, + "save_freq": 10000, + "checkpoint_format": "safetensors", + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "fused": null + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "parallelism": { + "dp_replicate": 1, + "dp_shard": 1, + "context_parallel": { + "ring_degree": 1, + "ulysses_degree": 1 + }, + "cfg_parallel": 1 + }, + "accelerator": { + "mixed_precision": "no", + "gradient_accumulation": { + "steps": 1 + }, + "grad_scaler": { + "init_scale": 65536.0, + "growth_factor": 2.0, + "backoff_factor": 0.5, + "growth_interval": 2000 + }, + "fsdp": { + "reshard_after_forward": true, + "wrap_modules": null, + "min_num_params": null, + "cpu_offload": false, + "ignored_modules": null + }, + "ddp": { + "find_unused_parameters": true, + "gradient_as_bucket_view": false, + "static_graph": false + }, + "compile": { + "enabled": false, + "backend": "inductor", + "mode": null, + "regional": true + }, + "activation_checkpointing": { + "mode": "none" + } + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": true, + "recording": false, + "recording_repo_id": null, + "recording_private": false + }, + "ema": { + "enable": false, + "decay": null, + "update_after_step": 0, + "inv_gamma": 1.0, + "power": 0.75, + "min_decay": 0.0, + "max_decay": 0.9999, + "use_for_eval": true + }, + "wandb": { + "enable": false, + "disable_artifact": false, + "project": "lerobot-nexarm", + "entity": null, + "notes": null, + "run_id": null, + "resume": null, + "mode": null, + "console": "wrap", + "console_multipart": false, + "console_chunk_max_seconds": 0, + "add_tags": true + }, + "peft": null, + "job": { + "target": null, + "image": "huggingface/lerobot-gpu:latest", + "timeout": "2d", + "detach": false, + "tags": [] + }, + "save_checkpoint_to_hub": false, + "sample_weighting": null, + "rename_map": {}, + "checkpoint_path": null +} \ No newline at end of file diff --git a/train_bsp10/checkpoints/050000/training_state/optimizer_param_groups.json b/train_bsp10/checkpoints/050000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..9695e26d3e848963853a1dfb787c5ee385cdde18 --- /dev/null +++ b/train_bsp10/checkpoints/050000/training_state/optimizer_param_groups.json @@ -0,0 +1,361 @@ +[ + { + "lr": 6.809148352279182e-06, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": false, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310, + 311, + 312, + 313, + 314, + 315, + 316, + 317, + 318, + 319, + 320, + 321, + 322, + 323, + 324, + 325, + 326, + 327, + 328, + 329, + 330, + 331, + 332, + 333, + 334, + 335, + 336, + 337, + 338, + 339 + ] + } +] \ No newline at end of file diff --git a/train_bsp10/checkpoints/050000/training_state/optimizer_state.safetensors b/train_bsp10/checkpoints/050000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..222974fa58f03f60b203dcc9b53a663929c2354b --- /dev/null +++ b/train_bsp10/checkpoints/050000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:afba1fb0f2843dffbda845a9aa4bc55d45f5252af717dd22cf7faf7aef163df5 +size 2341614992 diff --git a/train_bsp10/checkpoints/050000/training_state/rng_state.safetensors b/train_bsp10/checkpoints/050000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a24bcd6793e232dd5ced5b8ebacc47686f6189c0 --- /dev/null +++ b/train_bsp10/checkpoints/050000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfc7f809d9ebf3752ab7f8b50ad528674f029e37c284a3a84aa007082e4ecaad +size 15712 diff --git a/train_bsp10/checkpoints/050000/training_state/scheduler_state.json b/train_bsp10/checkpoints/050000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..3e59b68452b09f32fc0e0c1eb6738185ce9cada3 --- /dev/null +++ b/train_bsp10/checkpoints/050000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 50000, + "_step_count": 50001, + "_is_initial": false, + "_get_lr_called_within_step": false, + "_last_lr": [ + 6.809148352279182e-06 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/train_bsp10/checkpoints/050000/training_state/training_step.json b/train_bsp10/checkpoints/050000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..4208859fafb2985f0f94531a63bc9fb7ca9786bb --- /dev/null +++ b/train_bsp10/checkpoints/050000/training_state/training_step.json @@ -0,0 +1,13 @@ +{ + "step": 50000, + "dp_world_size": 1, + "batch_size": 32, + "grad_accum_steps": 1, + "mixed_precision": "no", + "parallelism": { + "dp_replicate": 1, + "dp_shard": 1, + "ring_degree": 1, + "ulysses_degree": 1 + } +} \ No newline at end of file diff --git a/train_bsp10/checkpoints/060000/pretrained_model/config.json b/train_bsp10/checkpoints/060000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..29162bf2c105e73be74560f4dce87a7c296ad7bf --- /dev/null +++ b/train_bsp10/checkpoints/060000/pretrained_model/config.json @@ -0,0 +1,122 @@ +{ + "type": "bspline", + "n_obs_steps": 2, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.front": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.wrist": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.top": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "dtype": null, + "use_amp": false, + "use_peft": false, + "push_to_hub": false, + "repo_id": null, + "private": null, + "tags": null, + "license": null, + "pretrained_path": null, + "pretrained_revision": null, + "horizon": 16, + "n_action_steps": 32, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "drop_n_last_frames": 0, + "vision_backbone": "resnet18", + "resize_shape": [ + 120, + 160 + ], + "crop_ratio": 1.0, + "crop_shape": null, + "crop_is_random": true, + "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": true, + "down_dims": [ + 512, + 1024, + 2048 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "gradient_checkpointing": false, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.5, + "num_inference_steps": 10, + "compile_model": false, + "compile_mode": "reduce-overhead", + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "bspline_degree": 3, + "bspline_chunk_size": 10, + "bspline_window": 64, + "bspline_max_error": 0.01, + "bspline_speed_up": 1.0, + "bspline_fit_workers": 0, + "bspline_mode": "diffusion", + "planbind_fine_spacing": 4, + "planbind_subdivision": 4, + "planbind_max_plan_age": 32, + "planbind_plan_noise": 0.02, + "planbind_replan_threshold": 0.0, + "planbind_latency": 0, + "planbind_commit": "latency", + "planbind_position_limit": 0.0, + "planbind_velocity_limit": 0.0 +} \ No newline at end of file diff --git a/train_bsp10/checkpoints/060000/pretrained_model/model.safetensors b/train_bsp10/checkpoints/060000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..bc4e2fe2249bf59555e21e23816944b6eb4d5a08 --- /dev/null +++ b/train_bsp10/checkpoints/060000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09319960d2f864466006e709b198f7fe75d1a744699c93fc14de1bf1c0e69b39 +size 1170940540 diff --git a/train_bsp10/checkpoints/060000/pretrained_model/policy_postprocessor.json b/train_bsp10/checkpoints/060000/pretrained_model/policy_postprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..f2c5e6cea7850bb999ab193f96021fa0129ce4db --- /dev/null +++ b/train_bsp10/checkpoints/060000/pretrained_model/policy_postprocessor.json @@ -0,0 +1,32 @@ +{ + "name": "policy_postprocessor", + "steps": [ + { + "registry_name": "unnormalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "norm_map": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + } + }, + "state_file": "policy_postprocessor_step_0_unnormalizer_processor.safetensors" + }, + { + "registry_name": "device_processor", + "config": { + "device": "cpu", + "float_dtype": null + } + } + ] +} \ No newline at end of file diff --git a/train_bsp10/checkpoints/060000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors b/train_bsp10/checkpoints/060000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ead5732518c212199e87eb8df6f0f49f32e15a1e --- /dev/null +++ b/train_bsp10/checkpoints/060000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55336fe114f19ef78b83ccc0926a3df66a9adc11a641f9adc0caf9c439ff8576 +size 8528 diff --git a/train_bsp10/checkpoints/060000/pretrained_model/policy_preprocessor.json b/train_bsp10/checkpoints/060000/pretrained_model/policy_preprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..981e0dfcb58027d80f783d8905452fadeaea830e --- /dev/null +++ b/train_bsp10/checkpoints/060000/pretrained_model/policy_preprocessor.json @@ -0,0 +1,72 @@ +{ + "name": "policy_preprocessor", + "steps": [ + { + "registry_name": "rename_observations_processor", + "config": { + "rename_map": {} + } + }, + { + "registry_name": "to_batch_processor", + "config": {} + }, + { + "registry_name": "device_processor", + "config": { + "device": "cuda", + "float_dtype": null + } + }, + { + "registry_name": "normalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.front": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.wrist": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.top": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "norm_map": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + } + }, + "state_file": "policy_preprocessor_step_3_normalizer_processor.safetensors" + } + ] +} \ No newline at end of file diff --git a/train_bsp10/checkpoints/060000/pretrained_model/policy_preprocessor_step_3_normalizer_processor.safetensors b/train_bsp10/checkpoints/060000/pretrained_model/policy_preprocessor_step_3_normalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f99bcf47b067091c10855f81f0663ca8d0f56ab8 --- /dev/null +++ b/train_bsp10/checkpoints/060000/pretrained_model/policy_preprocessor_step_3_normalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0f6cfea44fc7cfa8fa04ca706b89c90cce83136e90c3d0286428dfa2e1968d7 +size 8536 diff --git a/train_bsp10/checkpoints/060000/pretrained_model/train_config.json b/train_bsp10/checkpoints/060000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c074a87b77cd50d8d69e41de3aad8626a547e196 --- /dev/null +++ b/train_bsp10/checkpoints/060000/pretrained_model/train_config.json @@ -0,0 +1,347 @@ +{ + "dataset": { + "repo_id": "local/nexarm_stack_bowls", + "repo_type": "dataset", + "root": "/workspace/lerobot-nexarm/outputs/datasets/nexarm_stack_bowls", + "episodes": null, + "exclude_episodes": null, + "image_transforms": { + "enable": false, + "backend": "dataloader", + "gpu_compile": true, + "gpu_chunk_size": 32, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + }, + "affine": { + "weight": 1.0, + "type": "RandomAffine", + "kwargs": { + "degrees": [ + -5.0, + 5.0 + ], + "translate": [ + 0.05, + 0.05 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec", + "return_uint8": false, + "depth_output_unit": "mm", + "streaming": false, + "streaming_episode_pool_size": null, + "streaming_sampling_strategy": "remaining", + "streaming_prefetch_episodes": 8, + "streaming_byte_budget_gb": 8.0, + "streaming_decode_threads": 2, + "streaming_decoded_queue_size": 8, + "streaming_native_http_connections": null, + "streaming_native_http_subranges": 1, + "streaming_sidecar_lock_timeout_s": 1800, + "video_decoder_cache_size": null, + "eval_split": 0.0 + }, + "env": null, + "policy": { + "type": "bspline", + "n_obs_steps": 2, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.front": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.wrist": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.top": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "dtype": null, + "use_amp": false, + "use_peft": false, + "push_to_hub": false, + "repo_id": null, + "private": null, + "tags": null, + "license": null, + "pretrained_path": null, + "pretrained_revision": null, + "horizon": 16, + "n_action_steps": 32, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "drop_n_last_frames": 0, + "vision_backbone": "resnet18", + "resize_shape": [ + 120, + 160 + ], + "crop_ratio": 1.0, + "crop_shape": null, + "crop_is_random": true, + "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": true, + "down_dims": [ + 512, + 1024, + 2048 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "gradient_checkpointing": false, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.5, + "num_inference_steps": 10, + "compile_model": false, + "compile_mode": "reduce-overhead", + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "bspline_degree": 3, + "bspline_chunk_size": 10, + "bspline_window": 64, + "bspline_max_error": 0.01, + "bspline_speed_up": 1.0, + "bspline_fit_workers": 0, + "bspline_mode": "diffusion", + "planbind_fine_spacing": 4, + "planbind_subdivision": 4, + "planbind_max_plan_age": 32, + "planbind_plan_noise": 0.02, + "planbind_replan_threshold": 0.0, + "planbind_latency": 0, + "planbind_commit": "latency", + "planbind_position_limit": 0.0, + "planbind_velocity_limit": 0.0 + }, + "reward_model": null, + "output_dir": "/workspace/lerobot-nexarm/experiments/261027-plan-bind-two-step/outputs/full/train_bsp10", + "job_name": "261027-plan-bind-bsp10", + "resume": false, + "seed": 1000, + "cudnn_deterministic": false, + "num_workers": 16, + "batch_size": 32, + "prefetch_factor": 4, + "persistent_workers": true, + "dataloader_multiprocessing_context": "spawn", + "steps": 60000, + "env_eval_freq": 0, + "log_freq": 100, + "eval_steps": 0, + "max_eval_samples": 0, + "tolerance_s": 0.0001, + "save_checkpoint": true, + "save_freq": 10000, + "checkpoint_format": "safetensors", + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "fused": null + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "parallelism": { + "dp_replicate": 1, + "dp_shard": 1, + "context_parallel": { + "ring_degree": 1, + "ulysses_degree": 1 + }, + "cfg_parallel": 1 + }, + "accelerator": { + "mixed_precision": "no", + "gradient_accumulation": { + "steps": 1 + }, + "grad_scaler": { + "init_scale": 65536.0, + "growth_factor": 2.0, + "backoff_factor": 0.5, + "growth_interval": 2000 + }, + "fsdp": { + "reshard_after_forward": true, + "wrap_modules": null, + "min_num_params": null, + "cpu_offload": false, + "ignored_modules": null + }, + "ddp": { + "find_unused_parameters": true, + "gradient_as_bucket_view": false, + "static_graph": false + }, + "compile": { + "enabled": false, + "backend": "inductor", + "mode": null, + "regional": true + }, + "activation_checkpointing": { + "mode": "none" + } + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": true, + "recording": false, + "recording_repo_id": null, + "recording_private": false + }, + "ema": { + "enable": false, + "decay": null, + "update_after_step": 0, + "inv_gamma": 1.0, + "power": 0.75, + "min_decay": 0.0, + "max_decay": 0.9999, + "use_for_eval": true + }, + "wandb": { + "enable": false, + "disable_artifact": false, + "project": "lerobot-nexarm", + "entity": null, + "notes": null, + "run_id": null, + "resume": null, + "mode": null, + "console": "wrap", + "console_multipart": false, + "console_chunk_max_seconds": 0, + "add_tags": true + }, + "peft": null, + "job": { + "target": null, + "image": "huggingface/lerobot-gpu:latest", + "timeout": "2d", + "detach": false, + "tags": [] + }, + "save_checkpoint_to_hub": false, + "sample_weighting": null, + "rename_map": {}, + "checkpoint_path": null +} \ No newline at end of file diff --git a/train_bsp10/checkpoints/060000/training_state/optimizer_param_groups.json b/train_bsp10/checkpoints/060000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..befd92911309b87adcf9e11a8472b9db509a6130 --- /dev/null +++ b/train_bsp10/checkpoints/060000/training_state/optimizer_param_groups.json @@ -0,0 +1,361 @@ +[ + { + "lr": 0.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": false, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310, + 311, + 312, + 313, + 314, + 315, + 316, + 317, + 318, + 319, + 320, + 321, + 322, + 323, + 324, + 325, + 326, + 327, + 328, + 329, + 330, + 331, + 332, + 333, + 334, + 335, + 336, + 337, + 338, + 339 + ] + } +] \ No newline at end of file diff --git a/train_bsp10/checkpoints/060000/training_state/optimizer_state.safetensors b/train_bsp10/checkpoints/060000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1d13db8879516905493625db41bbb56a3c41eaad --- /dev/null +++ b/train_bsp10/checkpoints/060000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28ba6fb9cf4e4254a077c7f0073a85139184e85dce0bb408e412efe259ad2039 +size 2341614992 diff --git a/train_bsp10/checkpoints/060000/training_state/rng_state.safetensors b/train_bsp10/checkpoints/060000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ab5b79be900458cf1ccb25e94afb5fb3b7dd1f61 --- /dev/null +++ b/train_bsp10/checkpoints/060000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b246dc5151cd6acc6abf045df7b8d9e70d9403a4f58911d2bf351a7d5c9783a +size 15712 diff --git a/train_bsp10/checkpoints/060000/training_state/scheduler_state.json b/train_bsp10/checkpoints/060000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..b1d94a2da0059b2247243e3e57c356e0766e60e1 --- /dev/null +++ b/train_bsp10/checkpoints/060000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 60000, + "_step_count": 60001, + "_is_initial": false, + "_get_lr_called_within_step": false, + "_last_lr": [ + 0.0 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/train_bsp10/checkpoints/060000/training_state/training_step.json b/train_bsp10/checkpoints/060000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..126d1aab238e63998ab623bfa93c304b1070c5cf --- /dev/null +++ b/train_bsp10/checkpoints/060000/training_state/training_step.json @@ -0,0 +1,13 @@ +{ + "step": 60000, + "dp_world_size": 1, + "batch_size": 32, + "grad_accum_steps": 1, + "mixed_precision": "no", + "parallelism": { + "dp_replicate": 1, + "dp_shard": 1, + "ring_degree": 1, + "ulysses_degree": 1 + } +} \ No newline at end of file