diff --git a/all_checkpoints/010000/pretrained_model/config.json b/all_checkpoints/010000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..a3874265d1aff2ae2f35bf5c14af146414f9f125 --- /dev/null +++ b/all_checkpoints/010000/pretrained_model/config.json @@ -0,0 +1,106 @@ +{ + "type": "multi_task_dit", + "n_obs_steps": 2, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "observation.images.cam1": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + }, + "observation.images.cam2": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "device": "cuda", + "use_amp": false, + "use_peft": false, + "push_to_hub": false, + "repo_id": null, + "private": null, + "tags": null, + "license": null, + "pretrained_path": null, + "pretrained_revision": null, + "horizon": 32, + "n_action_steps": 24, + "objective": "flow_matching", + "noise_scheduler_type": "DDPM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": null, + "sigma_min": 0.0, + "num_integration_steps": 100, + "integration_method": "euler", + "timestep_sampling_strategy": "beta", + "timestep_sampling_s": 0.999, + "timestep_sampling_alpha": 1.5, + "timestep_sampling_beta": 1.0, + "hidden_dim": 512, + "num_layers": 6, + "num_heads": 8, + "dropout": 0.1, + "use_positional_encoding": false, + "timestep_embed_dim": 256, + "use_rope": true, + "rope_base": 10000.0, + "vision_encoder_name": "openai/clip-vit-base-patch16", + "use_separate_rgb_encoder_per_camera": false, + "vision_encoder_lr_multiplier": 0.1, + "image_resize_shape": [ + 224, + 224 + ], + "image_crop_shape": [ + 224, + 224 + ], + "image_crop_is_random": true, + "text_encoder_name": "openai/clip-vit-base-patch16", + "tokenizer_max_length": 77, + "tokenizer_padding": "max_length", + "tokenizer_padding_side": "right", + "tokenizer_truncation": true, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "optimizer_lr": 2e-05, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 0.0, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 0, + "do_mask_loss_for_padding": false, + "drop_n_last_frames": 7 +} \ No newline at end of file diff --git a/all_checkpoints/010000/pretrained_model/model.safetensors b/all_checkpoints/010000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5ac63083f4f15fb91cabbc21d995056ffeeb1b99 --- /dev/null +++ b/all_checkpoints/010000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:898da5d3ffa06e243548921ccb7e9a2e3bfdc7fc95d0833f4dbc1bd88228a29f +size 995640924 diff --git a/all_checkpoints/010000/pretrained_model/policy_postprocessor.json b/all_checkpoints/010000/pretrained_model/policy_postprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..0e896c35807a4a0616f52b341799a83165d6380e --- /dev/null +++ b/all_checkpoints/010000/pretrained_model/policy_postprocessor.json @@ -0,0 +1,32 @@ +{ + "name": "policy_postprocessor", + "steps": [ + { + "registry_name": "unnormalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "norm_map": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + } + }, + "state_file": "policy_postprocessor_step_0_unnormalizer_processor.safetensors" + }, + { + "registry_name": "device_processor", + "config": { + "device": "cpu", + "float_dtype": null + } + } + ] +} \ No newline at end of file diff --git a/all_checkpoints/010000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors b/all_checkpoints/010000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..05a9ab66dd9832301075a056e668bf83ae70aca5 --- /dev/null +++ b/all_checkpoints/010000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebcf8788ac2c38ccae8bc139e57a83da88fa38f6ab01a8fcc40cdbccdeb5286b +size 7608 diff --git a/all_checkpoints/010000/pretrained_model/policy_preprocessor.json b/all_checkpoints/010000/pretrained_model/policy_preprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..3254661686f66abc7b9baef1389d4ee961fcd9df --- /dev/null +++ b/all_checkpoints/010000/pretrained_model/policy_preprocessor.json @@ -0,0 +1,75 @@ +{ + "name": "policy_preprocessor", + "steps": [ + { + "registry_name": "rename_observations_processor", + "config": { + "rename_map": {} + } + }, + { + "registry_name": "to_batch_processor", + "config": {} + }, + { + "registry_name": "tokenizer_processor", + "config": { + "max_length": 77, + "task_key": "task", + "padding_side": "right", + "padding": "max_length", + "truncation": true, + "tokenizer_name": "openai/clip-vit-base-patch16" + } + }, + { + "registry_name": "device_processor", + "config": { + "device": "cuda", + "float_dtype": null + } + }, + { + "registry_name": "normalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "observation.state": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "observation.images.cam1": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + }, + "observation.images.cam2": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + }, + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "norm_map": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + } + }, + "state_file": "policy_preprocessor_step_4_normalizer_processor.safetensors" + } + ] +} \ No newline at end of file diff --git a/all_checkpoints/010000/pretrained_model/policy_preprocessor_step_4_normalizer_processor.safetensors b/all_checkpoints/010000/pretrained_model/policy_preprocessor_step_4_normalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..941aac9535d2ebd0a2779666c91e761ed89493e6 --- /dev/null +++ b/all_checkpoints/010000/pretrained_model/policy_preprocessor_step_4_normalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1483261062d5f6b8a80141ebb343170a30f06f9cda00c5b433b378d54610753c +size 7616 diff --git a/all_checkpoints/010000/pretrained_model/train_config.json b/all_checkpoints/010000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5d48afc7f96c2a80c152eba766f71b4dcd138706 --- /dev/null +++ b/all_checkpoints/010000/pretrained_model/train_config.json @@ -0,0 +1,257 @@ +{ + "dataset": { + "repo_id": "Bigenlight/cube_in_cup_lerobot_v3", + "root": "/home/junhyeong/workspace/youngwoong/cube_flow_matching/dataset", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + }, + "affine": { + "weight": 1.0, + "type": "RandomAffine", + "kwargs": { + "degrees": [ + -5.0, + 5.0 + ], + "translate": [ + 0.05, + 0.05 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec", + "return_uint8": false, + "depth_output_unit": "mm", + "streaming": false, + "eval_split": 0.13 + }, + "env": null, + "policy": { + "type": "multi_task_dit", + "n_obs_steps": 2, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "observation.images.cam1": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + }, + "observation.images.cam2": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "device": "cuda", + "use_amp": false, + "use_peft": false, + "push_to_hub": false, + "repo_id": null, + "private": null, + "tags": null, + "license": null, + "pretrained_path": null, + "pretrained_revision": null, + "horizon": 32, + "n_action_steps": 24, + "objective": "flow_matching", + "noise_scheduler_type": "DDPM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": null, + "sigma_min": 0.0, + "num_integration_steps": 100, + "integration_method": "euler", + "timestep_sampling_strategy": "beta", + "timestep_sampling_s": 0.999, + "timestep_sampling_alpha": 1.5, + "timestep_sampling_beta": 1.0, + "hidden_dim": 512, + "num_layers": 6, + "num_heads": 8, + "dropout": 0.1, + "use_positional_encoding": false, + "timestep_embed_dim": 256, + "use_rope": true, + "rope_base": 10000.0, + "vision_encoder_name": "openai/clip-vit-base-patch16", + "use_separate_rgb_encoder_per_camera": false, + "vision_encoder_lr_multiplier": 0.1, + "image_resize_shape": [ + 224, + 224 + ], + "image_crop_shape": [ + 224, + 224 + ], + "image_crop_is_random": true, + "text_encoder_name": "openai/clip-vit-base-patch16", + "tokenizer_max_length": 77, + "tokenizer_padding": "max_length", + "tokenizer_padding_side": "right", + "tokenizer_truncation": true, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "optimizer_lr": 2e-05, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 0.0, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 0, + "do_mask_loss_for_padding": false, + "drop_n_last_frames": 7 + }, + "reward_model": null, + "output_dir": "/home/junhyeong/workspace/youngwoong/cube_flow_matching/training/outputs/flow_matching_cube_in_cup", + "job_name": "flow_matching_cube_in_cup", + "resume": false, + "seed": 1000, + "cudnn_deterministic": false, + "num_workers": 4, + "batch_size": 8, + "prefetch_factor": 4, + "persistent_workers": true, + "steps": 100000, + "env_eval_freq": 20000, + "log_freq": 200, + "eval_steps": 2000, + "max_eval_samples": 0, + "tolerance_s": 0.0001, + "save_checkpoint": true, + "save_freq": 10000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 2e-05, + "weight_decay": 0.0, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 0, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 44, + "use_async_envs": true, + "recording": false, + "recording_repo_id": null, + "recording_private": false + }, + "wandb": { + "enable": false, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null, + "add_tags": true + }, + "peft": null, + "job": { + "target": null, + "image": "huggingface/lerobot-gpu:latest", + "timeout": "2d", + "detach": false, + "tags": [] + }, + "save_checkpoint_to_hub": false, + "sample_weighting": null, + "rename_map": {}, + "checkpoint_path": null +} \ No newline at end of file diff --git a/all_checkpoints/020000/pretrained_model/config.json b/all_checkpoints/020000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..a3874265d1aff2ae2f35bf5c14af146414f9f125 --- /dev/null +++ b/all_checkpoints/020000/pretrained_model/config.json @@ -0,0 +1,106 @@ +{ + "type": "multi_task_dit", + "n_obs_steps": 2, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "observation.images.cam1": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + }, + "observation.images.cam2": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "device": "cuda", + "use_amp": false, + "use_peft": false, + "push_to_hub": false, + "repo_id": null, + "private": null, + "tags": null, + "license": null, + "pretrained_path": null, + "pretrained_revision": null, + "horizon": 32, + "n_action_steps": 24, + "objective": "flow_matching", + "noise_scheduler_type": "DDPM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": null, + "sigma_min": 0.0, + "num_integration_steps": 100, + "integration_method": "euler", + "timestep_sampling_strategy": "beta", + "timestep_sampling_s": 0.999, + "timestep_sampling_alpha": 1.5, + "timestep_sampling_beta": 1.0, + "hidden_dim": 512, + "num_layers": 6, + "num_heads": 8, + "dropout": 0.1, + "use_positional_encoding": false, + "timestep_embed_dim": 256, + "use_rope": true, + "rope_base": 10000.0, + "vision_encoder_name": "openai/clip-vit-base-patch16", + "use_separate_rgb_encoder_per_camera": false, + "vision_encoder_lr_multiplier": 0.1, + "image_resize_shape": [ + 224, + 224 + ], + "image_crop_shape": [ + 224, + 224 + ], + "image_crop_is_random": true, + "text_encoder_name": "openai/clip-vit-base-patch16", + "tokenizer_max_length": 77, + "tokenizer_padding": "max_length", + "tokenizer_padding_side": "right", + "tokenizer_truncation": true, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "optimizer_lr": 2e-05, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 0.0, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 0, + "do_mask_loss_for_padding": false, + "drop_n_last_frames": 7 +} \ No newline at end of file diff --git a/all_checkpoints/020000/pretrained_model/model.safetensors b/all_checkpoints/020000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3d098cf1f257467ee231ca5137f1bd1774553c64 --- /dev/null +++ b/all_checkpoints/020000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f2e841ee1e3647b6f617fda40e95936db1eaa35747553860403ec481bd0a3d1 +size 995640924 diff --git a/all_checkpoints/020000/pretrained_model/policy_postprocessor.json b/all_checkpoints/020000/pretrained_model/policy_postprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..0e896c35807a4a0616f52b341799a83165d6380e --- /dev/null +++ b/all_checkpoints/020000/pretrained_model/policy_postprocessor.json @@ -0,0 +1,32 @@ +{ + "name": "policy_postprocessor", + "steps": [ + { + "registry_name": "unnormalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "norm_map": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + } + }, + "state_file": "policy_postprocessor_step_0_unnormalizer_processor.safetensors" + }, + { + "registry_name": "device_processor", + "config": { + "device": "cpu", + "float_dtype": null + } + } + ] +} \ No newline at end of file diff --git a/all_checkpoints/020000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors b/all_checkpoints/020000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..05a9ab66dd9832301075a056e668bf83ae70aca5 --- /dev/null +++ b/all_checkpoints/020000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebcf8788ac2c38ccae8bc139e57a83da88fa38f6ab01a8fcc40cdbccdeb5286b +size 7608 diff --git a/all_checkpoints/020000/pretrained_model/policy_preprocessor.json b/all_checkpoints/020000/pretrained_model/policy_preprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..3254661686f66abc7b9baef1389d4ee961fcd9df --- /dev/null +++ b/all_checkpoints/020000/pretrained_model/policy_preprocessor.json @@ -0,0 +1,75 @@ +{ + "name": "policy_preprocessor", + "steps": [ + { + "registry_name": "rename_observations_processor", + "config": { + "rename_map": {} + } + }, + { + "registry_name": "to_batch_processor", + "config": {} + }, + { + "registry_name": "tokenizer_processor", + "config": { + "max_length": 77, + "task_key": "task", + "padding_side": "right", + "padding": "max_length", + "truncation": true, + "tokenizer_name": "openai/clip-vit-base-patch16" + } + }, + { + "registry_name": "device_processor", + "config": { + "device": "cuda", + "float_dtype": null + } + }, + { + "registry_name": "normalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "observation.state": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "observation.images.cam1": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + }, + "observation.images.cam2": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + }, + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "norm_map": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + } + }, + "state_file": "policy_preprocessor_step_4_normalizer_processor.safetensors" + } + ] +} \ No newline at end of file diff --git a/all_checkpoints/020000/pretrained_model/policy_preprocessor_step_4_normalizer_processor.safetensors b/all_checkpoints/020000/pretrained_model/policy_preprocessor_step_4_normalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..941aac9535d2ebd0a2779666c91e761ed89493e6 --- /dev/null +++ b/all_checkpoints/020000/pretrained_model/policy_preprocessor_step_4_normalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1483261062d5f6b8a80141ebb343170a30f06f9cda00c5b433b378d54610753c +size 7616 diff --git a/all_checkpoints/020000/pretrained_model/train_config.json b/all_checkpoints/020000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5d48afc7f96c2a80c152eba766f71b4dcd138706 --- /dev/null +++ b/all_checkpoints/020000/pretrained_model/train_config.json @@ -0,0 +1,257 @@ +{ + "dataset": { + "repo_id": "Bigenlight/cube_in_cup_lerobot_v3", + "root": "/home/junhyeong/workspace/youngwoong/cube_flow_matching/dataset", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + }, + "affine": { + "weight": 1.0, + "type": "RandomAffine", + "kwargs": { + "degrees": [ + -5.0, + 5.0 + ], + "translate": [ + 0.05, + 0.05 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec", + "return_uint8": false, + "depth_output_unit": "mm", + "streaming": false, + "eval_split": 0.13 + }, + "env": null, + "policy": { + "type": "multi_task_dit", + "n_obs_steps": 2, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "observation.images.cam1": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + }, + "observation.images.cam2": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "device": "cuda", + "use_amp": false, + "use_peft": false, + "push_to_hub": false, + "repo_id": null, + "private": null, + "tags": null, + "license": null, + "pretrained_path": null, + "pretrained_revision": null, + "horizon": 32, + "n_action_steps": 24, + "objective": "flow_matching", + "noise_scheduler_type": "DDPM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": null, + "sigma_min": 0.0, + "num_integration_steps": 100, + "integration_method": "euler", + "timestep_sampling_strategy": "beta", + "timestep_sampling_s": 0.999, + "timestep_sampling_alpha": 1.5, + "timestep_sampling_beta": 1.0, + "hidden_dim": 512, + "num_layers": 6, + "num_heads": 8, + "dropout": 0.1, + "use_positional_encoding": false, + "timestep_embed_dim": 256, + "use_rope": true, + "rope_base": 10000.0, + "vision_encoder_name": "openai/clip-vit-base-patch16", + "use_separate_rgb_encoder_per_camera": false, + "vision_encoder_lr_multiplier": 0.1, + "image_resize_shape": [ + 224, + 224 + ], + "image_crop_shape": [ + 224, + 224 + ], + "image_crop_is_random": true, + "text_encoder_name": "openai/clip-vit-base-patch16", + "tokenizer_max_length": 77, + "tokenizer_padding": "max_length", + "tokenizer_padding_side": "right", + "tokenizer_truncation": true, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "optimizer_lr": 2e-05, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 0.0, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 0, + "do_mask_loss_for_padding": false, + "drop_n_last_frames": 7 + }, + "reward_model": null, + "output_dir": "/home/junhyeong/workspace/youngwoong/cube_flow_matching/training/outputs/flow_matching_cube_in_cup", + "job_name": "flow_matching_cube_in_cup", + "resume": false, + "seed": 1000, + "cudnn_deterministic": false, + "num_workers": 4, + "batch_size": 8, + "prefetch_factor": 4, + "persistent_workers": true, + "steps": 100000, + "env_eval_freq": 20000, + "log_freq": 200, + "eval_steps": 2000, + "max_eval_samples": 0, + "tolerance_s": 0.0001, + "save_checkpoint": true, + "save_freq": 10000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 2e-05, + "weight_decay": 0.0, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 0, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 44, + "use_async_envs": true, + "recording": false, + "recording_repo_id": null, + "recording_private": false + }, + "wandb": { + "enable": false, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null, + "add_tags": true + }, + "peft": null, + "job": { + "target": null, + "image": "huggingface/lerobot-gpu:latest", + "timeout": "2d", + "detach": false, + "tags": [] + }, + "save_checkpoint_to_hub": false, + "sample_weighting": null, + "rename_map": {}, + "checkpoint_path": null +} \ No newline at end of file diff --git a/all_checkpoints/030000/pretrained_model/config.json b/all_checkpoints/030000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..a3874265d1aff2ae2f35bf5c14af146414f9f125 --- /dev/null +++ b/all_checkpoints/030000/pretrained_model/config.json @@ -0,0 +1,106 @@ +{ + "type": "multi_task_dit", + "n_obs_steps": 2, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "observation.images.cam1": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + }, + "observation.images.cam2": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "device": "cuda", + "use_amp": false, + "use_peft": false, + "push_to_hub": false, + "repo_id": null, + "private": null, + "tags": null, + "license": null, + "pretrained_path": null, + "pretrained_revision": null, + "horizon": 32, + "n_action_steps": 24, + "objective": "flow_matching", + "noise_scheduler_type": "DDPM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": null, + "sigma_min": 0.0, + "num_integration_steps": 100, + "integration_method": "euler", + "timestep_sampling_strategy": "beta", + "timestep_sampling_s": 0.999, + "timestep_sampling_alpha": 1.5, + "timestep_sampling_beta": 1.0, + "hidden_dim": 512, + "num_layers": 6, + "num_heads": 8, + "dropout": 0.1, + "use_positional_encoding": false, + "timestep_embed_dim": 256, + "use_rope": true, + "rope_base": 10000.0, + "vision_encoder_name": "openai/clip-vit-base-patch16", + "use_separate_rgb_encoder_per_camera": false, + "vision_encoder_lr_multiplier": 0.1, + "image_resize_shape": [ + 224, + 224 + ], + "image_crop_shape": [ + 224, + 224 + ], + "image_crop_is_random": true, + "text_encoder_name": "openai/clip-vit-base-patch16", + "tokenizer_max_length": 77, + "tokenizer_padding": "max_length", + "tokenizer_padding_side": "right", + "tokenizer_truncation": true, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "optimizer_lr": 2e-05, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 0.0, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 0, + "do_mask_loss_for_padding": false, + "drop_n_last_frames": 7 +} \ No newline at end of file diff --git a/all_checkpoints/030000/pretrained_model/model.safetensors b/all_checkpoints/030000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9637d9b925c6c2a355e759fc616bff364ffea3cc --- /dev/null +++ b/all_checkpoints/030000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:693a9e33a0bfe305b4027742ee20795272ad4ef94ef879141f9bb883bad0e834 +size 995640924 diff --git a/all_checkpoints/030000/pretrained_model/policy_postprocessor.json b/all_checkpoints/030000/pretrained_model/policy_postprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..0e896c35807a4a0616f52b341799a83165d6380e --- /dev/null +++ b/all_checkpoints/030000/pretrained_model/policy_postprocessor.json @@ -0,0 +1,32 @@ +{ + "name": "policy_postprocessor", + "steps": [ + { + "registry_name": "unnormalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "norm_map": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + } + }, + "state_file": "policy_postprocessor_step_0_unnormalizer_processor.safetensors" + }, + { + "registry_name": "device_processor", + "config": { + "device": "cpu", + "float_dtype": null + } + } + ] +} \ No newline at end of file diff --git a/all_checkpoints/030000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors b/all_checkpoints/030000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..05a9ab66dd9832301075a056e668bf83ae70aca5 --- /dev/null +++ b/all_checkpoints/030000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebcf8788ac2c38ccae8bc139e57a83da88fa38f6ab01a8fcc40cdbccdeb5286b +size 7608 diff --git a/all_checkpoints/030000/pretrained_model/policy_preprocessor.json b/all_checkpoints/030000/pretrained_model/policy_preprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..3254661686f66abc7b9baef1389d4ee961fcd9df --- /dev/null +++ b/all_checkpoints/030000/pretrained_model/policy_preprocessor.json @@ -0,0 +1,75 @@ +{ + "name": "policy_preprocessor", + "steps": [ + { + "registry_name": "rename_observations_processor", + "config": { + "rename_map": {} + } + }, + { + "registry_name": "to_batch_processor", + "config": {} + }, + { + "registry_name": "tokenizer_processor", + "config": { + "max_length": 77, + "task_key": "task", + "padding_side": "right", + "padding": "max_length", + "truncation": true, + "tokenizer_name": "openai/clip-vit-base-patch16" + } + }, + { + "registry_name": "device_processor", + "config": { + "device": "cuda", + "float_dtype": null + } + }, + { + "registry_name": "normalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "observation.state": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "observation.images.cam1": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + }, + "observation.images.cam2": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + }, + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "norm_map": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + } + }, + "state_file": "policy_preprocessor_step_4_normalizer_processor.safetensors" + } + ] +} \ No newline at end of file diff --git a/all_checkpoints/030000/pretrained_model/policy_preprocessor_step_4_normalizer_processor.safetensors b/all_checkpoints/030000/pretrained_model/policy_preprocessor_step_4_normalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..941aac9535d2ebd0a2779666c91e761ed89493e6 --- /dev/null +++ b/all_checkpoints/030000/pretrained_model/policy_preprocessor_step_4_normalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1483261062d5f6b8a80141ebb343170a30f06f9cda00c5b433b378d54610753c +size 7616 diff --git a/all_checkpoints/030000/pretrained_model/train_config.json b/all_checkpoints/030000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5d48afc7f96c2a80c152eba766f71b4dcd138706 --- /dev/null +++ b/all_checkpoints/030000/pretrained_model/train_config.json @@ -0,0 +1,257 @@ +{ + "dataset": { + "repo_id": "Bigenlight/cube_in_cup_lerobot_v3", + "root": "/home/junhyeong/workspace/youngwoong/cube_flow_matching/dataset", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + }, + "affine": { + "weight": 1.0, + "type": "RandomAffine", + "kwargs": { + "degrees": [ + -5.0, + 5.0 + ], + "translate": [ + 0.05, + 0.05 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec", + "return_uint8": false, + "depth_output_unit": "mm", + "streaming": false, + "eval_split": 0.13 + }, + "env": null, + "policy": { + "type": "multi_task_dit", + "n_obs_steps": 2, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "observation.images.cam1": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + }, + "observation.images.cam2": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "device": "cuda", + "use_amp": false, + "use_peft": false, + "push_to_hub": false, + "repo_id": null, + "private": null, + "tags": null, + "license": null, + "pretrained_path": null, + "pretrained_revision": null, + "horizon": 32, + "n_action_steps": 24, + "objective": "flow_matching", + "noise_scheduler_type": "DDPM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": null, + "sigma_min": 0.0, + "num_integration_steps": 100, + "integration_method": "euler", + "timestep_sampling_strategy": "beta", + "timestep_sampling_s": 0.999, + "timestep_sampling_alpha": 1.5, + "timestep_sampling_beta": 1.0, + "hidden_dim": 512, + "num_layers": 6, + "num_heads": 8, + "dropout": 0.1, + "use_positional_encoding": false, + "timestep_embed_dim": 256, + "use_rope": true, + "rope_base": 10000.0, + "vision_encoder_name": "openai/clip-vit-base-patch16", + "use_separate_rgb_encoder_per_camera": false, + "vision_encoder_lr_multiplier": 0.1, + "image_resize_shape": [ + 224, + 224 + ], + "image_crop_shape": [ + 224, + 224 + ], + "image_crop_is_random": true, + "text_encoder_name": "openai/clip-vit-base-patch16", + "tokenizer_max_length": 77, + "tokenizer_padding": "max_length", + "tokenizer_padding_side": "right", + "tokenizer_truncation": true, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "optimizer_lr": 2e-05, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 0.0, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 0, + "do_mask_loss_for_padding": false, + "drop_n_last_frames": 7 + }, + "reward_model": null, + "output_dir": "/home/junhyeong/workspace/youngwoong/cube_flow_matching/training/outputs/flow_matching_cube_in_cup", + "job_name": "flow_matching_cube_in_cup", + "resume": false, + "seed": 1000, + "cudnn_deterministic": false, + "num_workers": 4, + "batch_size": 8, + "prefetch_factor": 4, + "persistent_workers": true, + "steps": 100000, + "env_eval_freq": 20000, + "log_freq": 200, + "eval_steps": 2000, + "max_eval_samples": 0, + "tolerance_s": 0.0001, + "save_checkpoint": true, + "save_freq": 10000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 2e-05, + "weight_decay": 0.0, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 0, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 44, + "use_async_envs": true, + "recording": false, + "recording_repo_id": null, + "recording_private": false + }, + "wandb": { + "enable": false, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null, + "add_tags": true + }, + "peft": null, + "job": { + "target": null, + "image": "huggingface/lerobot-gpu:latest", + "timeout": "2d", + "detach": false, + "tags": [] + }, + "save_checkpoint_to_hub": false, + "sample_weighting": null, + "rename_map": {}, + "checkpoint_path": null +} \ No newline at end of file diff --git a/all_checkpoints/040000/pretrained_model/config.json b/all_checkpoints/040000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..a3874265d1aff2ae2f35bf5c14af146414f9f125 --- /dev/null +++ b/all_checkpoints/040000/pretrained_model/config.json @@ -0,0 +1,106 @@ +{ + "type": "multi_task_dit", + "n_obs_steps": 2, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "observation.images.cam1": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + }, + "observation.images.cam2": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "device": "cuda", + "use_amp": false, + "use_peft": false, + "push_to_hub": false, + "repo_id": null, + "private": null, + "tags": null, + "license": null, + "pretrained_path": null, + "pretrained_revision": null, + "horizon": 32, + "n_action_steps": 24, + "objective": "flow_matching", + "noise_scheduler_type": "DDPM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": null, + "sigma_min": 0.0, + "num_integration_steps": 100, + "integration_method": "euler", + "timestep_sampling_strategy": "beta", + "timestep_sampling_s": 0.999, + "timestep_sampling_alpha": 1.5, + "timestep_sampling_beta": 1.0, + "hidden_dim": 512, + "num_layers": 6, + "num_heads": 8, + "dropout": 0.1, + "use_positional_encoding": false, + "timestep_embed_dim": 256, + "use_rope": true, + "rope_base": 10000.0, + "vision_encoder_name": "openai/clip-vit-base-patch16", + "use_separate_rgb_encoder_per_camera": false, + "vision_encoder_lr_multiplier": 0.1, + "image_resize_shape": [ + 224, + 224 + ], + "image_crop_shape": [ + 224, + 224 + ], + "image_crop_is_random": true, + "text_encoder_name": "openai/clip-vit-base-patch16", + "tokenizer_max_length": 77, + "tokenizer_padding": "max_length", + "tokenizer_padding_side": "right", + "tokenizer_truncation": true, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "optimizer_lr": 2e-05, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 0.0, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 0, + "do_mask_loss_for_padding": false, + "drop_n_last_frames": 7 +} \ No newline at end of file diff --git a/all_checkpoints/040000/pretrained_model/model.safetensors b/all_checkpoints/040000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7aacdcf2f6801d8530d5d3e5a6ae152ea4a0cff0 --- /dev/null +++ b/all_checkpoints/040000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:626bf3c339291729d8aca477e5c7c86a94f611189d6efd6ccfcb4c16de015e4e +size 995640924 diff --git a/all_checkpoints/040000/pretrained_model/policy_postprocessor.json b/all_checkpoints/040000/pretrained_model/policy_postprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..0e896c35807a4a0616f52b341799a83165d6380e --- /dev/null +++ b/all_checkpoints/040000/pretrained_model/policy_postprocessor.json @@ -0,0 +1,32 @@ +{ + "name": "policy_postprocessor", + "steps": [ + { + "registry_name": "unnormalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "norm_map": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + } + }, + "state_file": "policy_postprocessor_step_0_unnormalizer_processor.safetensors" + }, + { + "registry_name": "device_processor", + "config": { + "device": "cpu", + "float_dtype": null + } + } + ] +} \ No newline at end of file diff --git a/all_checkpoints/040000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors b/all_checkpoints/040000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..05a9ab66dd9832301075a056e668bf83ae70aca5 --- /dev/null +++ b/all_checkpoints/040000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebcf8788ac2c38ccae8bc139e57a83da88fa38f6ab01a8fcc40cdbccdeb5286b +size 7608 diff --git a/all_checkpoints/040000/pretrained_model/policy_preprocessor.json b/all_checkpoints/040000/pretrained_model/policy_preprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..3254661686f66abc7b9baef1389d4ee961fcd9df --- /dev/null +++ b/all_checkpoints/040000/pretrained_model/policy_preprocessor.json @@ -0,0 +1,75 @@ +{ + "name": "policy_preprocessor", + "steps": [ + { + "registry_name": "rename_observations_processor", + "config": { + "rename_map": {} + } + }, + { + "registry_name": "to_batch_processor", + "config": {} + }, + { + "registry_name": "tokenizer_processor", + "config": { + "max_length": 77, + "task_key": "task", + "padding_side": "right", + "padding": "max_length", + "truncation": true, + "tokenizer_name": "openai/clip-vit-base-patch16" + } + }, + { + "registry_name": "device_processor", + "config": { + "device": "cuda", + "float_dtype": null + } + }, + { + "registry_name": "normalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "observation.state": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "observation.images.cam1": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + }, + "observation.images.cam2": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + }, + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "norm_map": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + } + }, + "state_file": "policy_preprocessor_step_4_normalizer_processor.safetensors" + } + ] +} \ No newline at end of file diff --git a/all_checkpoints/040000/pretrained_model/policy_preprocessor_step_4_normalizer_processor.safetensors b/all_checkpoints/040000/pretrained_model/policy_preprocessor_step_4_normalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..941aac9535d2ebd0a2779666c91e761ed89493e6 --- /dev/null +++ b/all_checkpoints/040000/pretrained_model/policy_preprocessor_step_4_normalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1483261062d5f6b8a80141ebb343170a30f06f9cda00c5b433b378d54610753c +size 7616 diff --git a/all_checkpoints/040000/pretrained_model/train_config.json b/all_checkpoints/040000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5d48afc7f96c2a80c152eba766f71b4dcd138706 --- /dev/null +++ b/all_checkpoints/040000/pretrained_model/train_config.json @@ -0,0 +1,257 @@ +{ + "dataset": { + "repo_id": "Bigenlight/cube_in_cup_lerobot_v3", + "root": "/home/junhyeong/workspace/youngwoong/cube_flow_matching/dataset", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + }, + "affine": { + "weight": 1.0, + "type": "RandomAffine", + "kwargs": { + "degrees": [ + -5.0, + 5.0 + ], + "translate": [ + 0.05, + 0.05 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec", + "return_uint8": false, + "depth_output_unit": "mm", + "streaming": false, + "eval_split": 0.13 + }, + "env": null, + "policy": { + "type": "multi_task_dit", + "n_obs_steps": 2, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "observation.images.cam1": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + }, + "observation.images.cam2": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "device": "cuda", + "use_amp": false, + "use_peft": false, + "push_to_hub": false, + "repo_id": null, + "private": null, + "tags": null, + "license": null, + "pretrained_path": null, + "pretrained_revision": null, + "horizon": 32, + "n_action_steps": 24, + "objective": "flow_matching", + "noise_scheduler_type": "DDPM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": null, + "sigma_min": 0.0, + "num_integration_steps": 100, + "integration_method": "euler", + "timestep_sampling_strategy": "beta", + "timestep_sampling_s": 0.999, + "timestep_sampling_alpha": 1.5, + "timestep_sampling_beta": 1.0, + "hidden_dim": 512, + "num_layers": 6, + "num_heads": 8, + "dropout": 0.1, + "use_positional_encoding": false, + "timestep_embed_dim": 256, + "use_rope": true, + "rope_base": 10000.0, + "vision_encoder_name": "openai/clip-vit-base-patch16", + "use_separate_rgb_encoder_per_camera": false, + "vision_encoder_lr_multiplier": 0.1, + "image_resize_shape": [ + 224, + 224 + ], + "image_crop_shape": [ + 224, + 224 + ], + "image_crop_is_random": true, + "text_encoder_name": "openai/clip-vit-base-patch16", + "tokenizer_max_length": 77, + "tokenizer_padding": "max_length", + "tokenizer_padding_side": "right", + "tokenizer_truncation": true, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "optimizer_lr": 2e-05, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 0.0, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 0, + "do_mask_loss_for_padding": false, + "drop_n_last_frames": 7 + }, + "reward_model": null, + "output_dir": "/home/junhyeong/workspace/youngwoong/cube_flow_matching/training/outputs/flow_matching_cube_in_cup", + "job_name": "flow_matching_cube_in_cup", + "resume": false, + "seed": 1000, + "cudnn_deterministic": false, + "num_workers": 4, + "batch_size": 8, + "prefetch_factor": 4, + "persistent_workers": true, + "steps": 100000, + "env_eval_freq": 20000, + "log_freq": 200, + "eval_steps": 2000, + "max_eval_samples": 0, + "tolerance_s": 0.0001, + "save_checkpoint": true, + "save_freq": 10000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 2e-05, + "weight_decay": 0.0, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 0, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 44, + "use_async_envs": true, + "recording": false, + "recording_repo_id": null, + "recording_private": false + }, + "wandb": { + "enable": false, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null, + "add_tags": true + }, + "peft": null, + "job": { + "target": null, + "image": "huggingface/lerobot-gpu:latest", + "timeout": "2d", + "detach": false, + "tags": [] + }, + "save_checkpoint_to_hub": false, + "sample_weighting": null, + "rename_map": {}, + "checkpoint_path": null +} \ No newline at end of file diff --git a/all_checkpoints/050000/pretrained_model/config.json b/all_checkpoints/050000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..a3874265d1aff2ae2f35bf5c14af146414f9f125 --- /dev/null +++ b/all_checkpoints/050000/pretrained_model/config.json @@ -0,0 +1,106 @@ +{ + "type": "multi_task_dit", + "n_obs_steps": 2, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "observation.images.cam1": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + }, + "observation.images.cam2": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "device": "cuda", + "use_amp": false, + "use_peft": false, + "push_to_hub": false, + "repo_id": null, + "private": null, + "tags": null, + "license": null, + "pretrained_path": null, + "pretrained_revision": null, + "horizon": 32, + "n_action_steps": 24, + "objective": "flow_matching", + "noise_scheduler_type": "DDPM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": null, + "sigma_min": 0.0, + "num_integration_steps": 100, + "integration_method": "euler", + "timestep_sampling_strategy": "beta", + "timestep_sampling_s": 0.999, + "timestep_sampling_alpha": 1.5, + "timestep_sampling_beta": 1.0, + "hidden_dim": 512, + "num_layers": 6, + "num_heads": 8, + "dropout": 0.1, + "use_positional_encoding": false, + "timestep_embed_dim": 256, + "use_rope": true, + "rope_base": 10000.0, + "vision_encoder_name": "openai/clip-vit-base-patch16", + "use_separate_rgb_encoder_per_camera": false, + "vision_encoder_lr_multiplier": 0.1, + "image_resize_shape": [ + 224, + 224 + ], + "image_crop_shape": [ + 224, + 224 + ], + "image_crop_is_random": true, + "text_encoder_name": "openai/clip-vit-base-patch16", + "tokenizer_max_length": 77, + "tokenizer_padding": "max_length", + "tokenizer_padding_side": "right", + "tokenizer_truncation": true, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "optimizer_lr": 2e-05, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 0.0, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 0, + "do_mask_loss_for_padding": false, + "drop_n_last_frames": 7 +} \ No newline at end of file diff --git a/all_checkpoints/050000/pretrained_model/model.safetensors b/all_checkpoints/050000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..dfa31fd582875edaeb3303cfa15d957db9361b21 --- /dev/null +++ b/all_checkpoints/050000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0459e22000f1c311da7b9162b33fbaa6cf8c7c7eb887ce0e7a893cba68914348 +size 995640924 diff --git a/all_checkpoints/050000/pretrained_model/policy_postprocessor.json b/all_checkpoints/050000/pretrained_model/policy_postprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..0e896c35807a4a0616f52b341799a83165d6380e --- /dev/null +++ b/all_checkpoints/050000/pretrained_model/policy_postprocessor.json @@ -0,0 +1,32 @@ +{ + "name": "policy_postprocessor", + "steps": [ + { + "registry_name": "unnormalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "norm_map": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + } + }, + "state_file": "policy_postprocessor_step_0_unnormalizer_processor.safetensors" + }, + { + "registry_name": "device_processor", + "config": { + "device": "cpu", + "float_dtype": null + } + } + ] +} \ No newline at end of file diff --git a/all_checkpoints/050000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors b/all_checkpoints/050000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..05a9ab66dd9832301075a056e668bf83ae70aca5 --- /dev/null +++ b/all_checkpoints/050000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebcf8788ac2c38ccae8bc139e57a83da88fa38f6ab01a8fcc40cdbccdeb5286b +size 7608 diff --git a/all_checkpoints/050000/pretrained_model/policy_preprocessor.json b/all_checkpoints/050000/pretrained_model/policy_preprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..3254661686f66abc7b9baef1389d4ee961fcd9df --- /dev/null +++ b/all_checkpoints/050000/pretrained_model/policy_preprocessor.json @@ -0,0 +1,75 @@ +{ + "name": "policy_preprocessor", + "steps": [ + { + "registry_name": "rename_observations_processor", + "config": { + "rename_map": {} + } + }, + { + "registry_name": "to_batch_processor", + "config": {} + }, + { + "registry_name": "tokenizer_processor", + "config": { + "max_length": 77, + "task_key": "task", + "padding_side": "right", + "padding": "max_length", + "truncation": true, + "tokenizer_name": "openai/clip-vit-base-patch16" + } + }, + { + "registry_name": "device_processor", + "config": { + "device": "cuda", + "float_dtype": null + } + }, + { + "registry_name": "normalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "observation.state": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "observation.images.cam1": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + }, + "observation.images.cam2": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + }, + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "norm_map": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + } + }, + "state_file": "policy_preprocessor_step_4_normalizer_processor.safetensors" + } + ] +} \ No newline at end of file diff --git a/all_checkpoints/050000/pretrained_model/policy_preprocessor_step_4_normalizer_processor.safetensors b/all_checkpoints/050000/pretrained_model/policy_preprocessor_step_4_normalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..941aac9535d2ebd0a2779666c91e761ed89493e6 --- /dev/null +++ b/all_checkpoints/050000/pretrained_model/policy_preprocessor_step_4_normalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1483261062d5f6b8a80141ebb343170a30f06f9cda00c5b433b378d54610753c +size 7616 diff --git a/all_checkpoints/050000/pretrained_model/train_config.json b/all_checkpoints/050000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5d48afc7f96c2a80c152eba766f71b4dcd138706 --- /dev/null +++ b/all_checkpoints/050000/pretrained_model/train_config.json @@ -0,0 +1,257 @@ +{ + "dataset": { + "repo_id": "Bigenlight/cube_in_cup_lerobot_v3", + "root": "/home/junhyeong/workspace/youngwoong/cube_flow_matching/dataset", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + }, + "affine": { + "weight": 1.0, + "type": "RandomAffine", + "kwargs": { + "degrees": [ + -5.0, + 5.0 + ], + "translate": [ + 0.05, + 0.05 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec", + "return_uint8": false, + "depth_output_unit": "mm", + "streaming": false, + "eval_split": 0.13 + }, + "env": null, + "policy": { + "type": "multi_task_dit", + "n_obs_steps": 2, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "observation.images.cam1": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + }, + "observation.images.cam2": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "device": "cuda", + "use_amp": false, + "use_peft": false, + "push_to_hub": false, + "repo_id": null, + "private": null, + "tags": null, + "license": null, + "pretrained_path": null, + "pretrained_revision": null, + "horizon": 32, + "n_action_steps": 24, + "objective": "flow_matching", + "noise_scheduler_type": "DDPM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": null, + "sigma_min": 0.0, + "num_integration_steps": 100, + "integration_method": "euler", + "timestep_sampling_strategy": "beta", + "timestep_sampling_s": 0.999, + "timestep_sampling_alpha": 1.5, + "timestep_sampling_beta": 1.0, + "hidden_dim": 512, + "num_layers": 6, + "num_heads": 8, + "dropout": 0.1, + "use_positional_encoding": false, + "timestep_embed_dim": 256, + "use_rope": true, + "rope_base": 10000.0, + "vision_encoder_name": "openai/clip-vit-base-patch16", + "use_separate_rgb_encoder_per_camera": false, + "vision_encoder_lr_multiplier": 0.1, + "image_resize_shape": [ + 224, + 224 + ], + "image_crop_shape": [ + 224, + 224 + ], + "image_crop_is_random": true, + "text_encoder_name": "openai/clip-vit-base-patch16", + "tokenizer_max_length": 77, + "tokenizer_padding": "max_length", + "tokenizer_padding_side": "right", + "tokenizer_truncation": true, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "optimizer_lr": 2e-05, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 0.0, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 0, + "do_mask_loss_for_padding": false, + "drop_n_last_frames": 7 + }, + "reward_model": null, + "output_dir": "/home/junhyeong/workspace/youngwoong/cube_flow_matching/training/outputs/flow_matching_cube_in_cup", + "job_name": "flow_matching_cube_in_cup", + "resume": false, + "seed": 1000, + "cudnn_deterministic": false, + "num_workers": 4, + "batch_size": 8, + "prefetch_factor": 4, + "persistent_workers": true, + "steps": 100000, + "env_eval_freq": 20000, + "log_freq": 200, + "eval_steps": 2000, + "max_eval_samples": 0, + "tolerance_s": 0.0001, + "save_checkpoint": true, + "save_freq": 10000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 2e-05, + "weight_decay": 0.0, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 0, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 44, + "use_async_envs": true, + "recording": false, + "recording_repo_id": null, + "recording_private": false + }, + "wandb": { + "enable": false, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null, + "add_tags": true + }, + "peft": null, + "job": { + "target": null, + "image": "huggingface/lerobot-gpu:latest", + "timeout": "2d", + "detach": false, + "tags": [] + }, + "save_checkpoint_to_hub": false, + "sample_weighting": null, + "rename_map": {}, + "checkpoint_path": null +} \ No newline at end of file diff --git a/all_checkpoints/060000/pretrained_model/config.json b/all_checkpoints/060000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..a3874265d1aff2ae2f35bf5c14af146414f9f125 --- /dev/null +++ b/all_checkpoints/060000/pretrained_model/config.json @@ -0,0 +1,106 @@ +{ + "type": "multi_task_dit", + "n_obs_steps": 2, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "observation.images.cam1": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + }, + "observation.images.cam2": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "device": "cuda", + "use_amp": false, + "use_peft": false, + "push_to_hub": false, + "repo_id": null, + "private": null, + "tags": null, + "license": null, + "pretrained_path": null, + "pretrained_revision": null, + "horizon": 32, + "n_action_steps": 24, + "objective": "flow_matching", + "noise_scheduler_type": "DDPM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": null, + "sigma_min": 0.0, + "num_integration_steps": 100, + "integration_method": "euler", + "timestep_sampling_strategy": "beta", + "timestep_sampling_s": 0.999, + "timestep_sampling_alpha": 1.5, + "timestep_sampling_beta": 1.0, + "hidden_dim": 512, + "num_layers": 6, + "num_heads": 8, + "dropout": 0.1, + "use_positional_encoding": false, + "timestep_embed_dim": 256, + "use_rope": true, + "rope_base": 10000.0, + "vision_encoder_name": "openai/clip-vit-base-patch16", + "use_separate_rgb_encoder_per_camera": false, + "vision_encoder_lr_multiplier": 0.1, + "image_resize_shape": [ + 224, + 224 + ], + "image_crop_shape": [ + 224, + 224 + ], + "image_crop_is_random": true, + "text_encoder_name": "openai/clip-vit-base-patch16", + "tokenizer_max_length": 77, + "tokenizer_padding": "max_length", + "tokenizer_padding_side": "right", + "tokenizer_truncation": true, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "optimizer_lr": 2e-05, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 0.0, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 0, + "do_mask_loss_for_padding": false, + "drop_n_last_frames": 7 +} \ No newline at end of file diff --git a/all_checkpoints/060000/pretrained_model/model.safetensors b/all_checkpoints/060000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0d4eb7711cd452663716d81139f55b1d53a0ce17 --- /dev/null +++ b/all_checkpoints/060000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a172e7b65f29500afe3d0ccca3ee5b9c0861f3c521c03da19cd6870ba0e0ec80 +size 995640924 diff --git a/all_checkpoints/060000/pretrained_model/policy_postprocessor.json b/all_checkpoints/060000/pretrained_model/policy_postprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..0e896c35807a4a0616f52b341799a83165d6380e --- /dev/null +++ b/all_checkpoints/060000/pretrained_model/policy_postprocessor.json @@ -0,0 +1,32 @@ +{ + "name": "policy_postprocessor", + "steps": [ + { + "registry_name": "unnormalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "norm_map": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + } + }, + "state_file": "policy_postprocessor_step_0_unnormalizer_processor.safetensors" + }, + { + "registry_name": "device_processor", + "config": { + "device": "cpu", + "float_dtype": null + } + } + ] +} \ No newline at end of file diff --git a/all_checkpoints/060000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors b/all_checkpoints/060000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..05a9ab66dd9832301075a056e668bf83ae70aca5 --- /dev/null +++ b/all_checkpoints/060000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebcf8788ac2c38ccae8bc139e57a83da88fa38f6ab01a8fcc40cdbccdeb5286b +size 7608 diff --git a/all_checkpoints/060000/pretrained_model/policy_preprocessor.json b/all_checkpoints/060000/pretrained_model/policy_preprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..3254661686f66abc7b9baef1389d4ee961fcd9df --- /dev/null +++ b/all_checkpoints/060000/pretrained_model/policy_preprocessor.json @@ -0,0 +1,75 @@ +{ + "name": "policy_preprocessor", + "steps": [ + { + "registry_name": "rename_observations_processor", + "config": { + "rename_map": {} + } + }, + { + "registry_name": "to_batch_processor", + "config": {} + }, + { + "registry_name": "tokenizer_processor", + "config": { + "max_length": 77, + "task_key": "task", + "padding_side": "right", + "padding": "max_length", + "truncation": true, + "tokenizer_name": "openai/clip-vit-base-patch16" + } + }, + { + "registry_name": "device_processor", + "config": { + "device": "cuda", + "float_dtype": null + } + }, + { + "registry_name": "normalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "observation.state": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "observation.images.cam1": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + }, + "observation.images.cam2": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + }, + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "norm_map": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + } + }, + "state_file": "policy_preprocessor_step_4_normalizer_processor.safetensors" + } + ] +} \ No newline at end of file diff --git a/all_checkpoints/060000/pretrained_model/policy_preprocessor_step_4_normalizer_processor.safetensors b/all_checkpoints/060000/pretrained_model/policy_preprocessor_step_4_normalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..941aac9535d2ebd0a2779666c91e761ed89493e6 --- /dev/null +++ b/all_checkpoints/060000/pretrained_model/policy_preprocessor_step_4_normalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1483261062d5f6b8a80141ebb343170a30f06f9cda00c5b433b378d54610753c +size 7616 diff --git a/all_checkpoints/060000/pretrained_model/train_config.json b/all_checkpoints/060000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5d48afc7f96c2a80c152eba766f71b4dcd138706 --- /dev/null +++ b/all_checkpoints/060000/pretrained_model/train_config.json @@ -0,0 +1,257 @@ +{ + "dataset": { + "repo_id": "Bigenlight/cube_in_cup_lerobot_v3", + "root": "/home/junhyeong/workspace/youngwoong/cube_flow_matching/dataset", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + }, + "affine": { + "weight": 1.0, + "type": "RandomAffine", + "kwargs": { + "degrees": [ + -5.0, + 5.0 + ], + "translate": [ + 0.05, + 0.05 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec", + "return_uint8": false, + "depth_output_unit": "mm", + "streaming": false, + "eval_split": 0.13 + }, + "env": null, + "policy": { + "type": "multi_task_dit", + "n_obs_steps": 2, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "observation.images.cam1": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + }, + "observation.images.cam2": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "device": "cuda", + "use_amp": false, + "use_peft": false, + "push_to_hub": false, + "repo_id": null, + "private": null, + "tags": null, + "license": null, + "pretrained_path": null, + "pretrained_revision": null, + "horizon": 32, + "n_action_steps": 24, + "objective": "flow_matching", + "noise_scheduler_type": "DDPM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": null, + "sigma_min": 0.0, + "num_integration_steps": 100, + "integration_method": "euler", + "timestep_sampling_strategy": "beta", + "timestep_sampling_s": 0.999, + "timestep_sampling_alpha": 1.5, + "timestep_sampling_beta": 1.0, + "hidden_dim": 512, + "num_layers": 6, + "num_heads": 8, + "dropout": 0.1, + "use_positional_encoding": false, + "timestep_embed_dim": 256, + "use_rope": true, + "rope_base": 10000.0, + "vision_encoder_name": "openai/clip-vit-base-patch16", + "use_separate_rgb_encoder_per_camera": false, + "vision_encoder_lr_multiplier": 0.1, + "image_resize_shape": [ + 224, + 224 + ], + "image_crop_shape": [ + 224, + 224 + ], + "image_crop_is_random": true, + "text_encoder_name": "openai/clip-vit-base-patch16", + "tokenizer_max_length": 77, + "tokenizer_padding": "max_length", + "tokenizer_padding_side": "right", + "tokenizer_truncation": true, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "optimizer_lr": 2e-05, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 0.0, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 0, + "do_mask_loss_for_padding": false, + "drop_n_last_frames": 7 + }, + "reward_model": null, + "output_dir": "/home/junhyeong/workspace/youngwoong/cube_flow_matching/training/outputs/flow_matching_cube_in_cup", + "job_name": "flow_matching_cube_in_cup", + "resume": false, + "seed": 1000, + "cudnn_deterministic": false, + "num_workers": 4, + "batch_size": 8, + "prefetch_factor": 4, + "persistent_workers": true, + "steps": 100000, + "env_eval_freq": 20000, + "log_freq": 200, + "eval_steps": 2000, + "max_eval_samples": 0, + "tolerance_s": 0.0001, + "save_checkpoint": true, + "save_freq": 10000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 2e-05, + "weight_decay": 0.0, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 0, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 44, + "use_async_envs": true, + "recording": false, + "recording_repo_id": null, + "recording_private": false + }, + "wandb": { + "enable": false, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null, + "add_tags": true + }, + "peft": null, + "job": { + "target": null, + "image": "huggingface/lerobot-gpu:latest", + "timeout": "2d", + "detach": false, + "tags": [] + }, + "save_checkpoint_to_hub": false, + "sample_weighting": null, + "rename_map": {}, + "checkpoint_path": null +} \ No newline at end of file diff --git a/all_checkpoints/070000/pretrained_model/config.json b/all_checkpoints/070000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..a3874265d1aff2ae2f35bf5c14af146414f9f125 --- /dev/null +++ b/all_checkpoints/070000/pretrained_model/config.json @@ -0,0 +1,106 @@ +{ + "type": "multi_task_dit", + "n_obs_steps": 2, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "observation.images.cam1": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + }, + "observation.images.cam2": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "device": "cuda", + "use_amp": false, + "use_peft": false, + "push_to_hub": false, + "repo_id": null, + "private": null, + "tags": null, + "license": null, + "pretrained_path": null, + "pretrained_revision": null, + "horizon": 32, + "n_action_steps": 24, + "objective": "flow_matching", + "noise_scheduler_type": "DDPM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": null, + "sigma_min": 0.0, + "num_integration_steps": 100, + "integration_method": "euler", + "timestep_sampling_strategy": "beta", + "timestep_sampling_s": 0.999, + "timestep_sampling_alpha": 1.5, + "timestep_sampling_beta": 1.0, + "hidden_dim": 512, + "num_layers": 6, + "num_heads": 8, + "dropout": 0.1, + "use_positional_encoding": false, + "timestep_embed_dim": 256, + "use_rope": true, + "rope_base": 10000.0, + "vision_encoder_name": "openai/clip-vit-base-patch16", + "use_separate_rgb_encoder_per_camera": false, + "vision_encoder_lr_multiplier": 0.1, + "image_resize_shape": [ + 224, + 224 + ], + "image_crop_shape": [ + 224, + 224 + ], + "image_crop_is_random": true, + "text_encoder_name": "openai/clip-vit-base-patch16", + "tokenizer_max_length": 77, + "tokenizer_padding": "max_length", + "tokenizer_padding_side": "right", + "tokenizer_truncation": true, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "optimizer_lr": 2e-05, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 0.0, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 0, + "do_mask_loss_for_padding": false, + "drop_n_last_frames": 7 +} \ No newline at end of file diff --git a/all_checkpoints/070000/pretrained_model/model.safetensors b/all_checkpoints/070000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2066c6d8e9c7e2c4444591f63975ba6486bf9371 --- /dev/null +++ b/all_checkpoints/070000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5cd59eacaf671e62e28df4691ea39c98c1e5000db5daa52de8d55dbe7e86aac +size 995640924 diff --git a/all_checkpoints/070000/pretrained_model/policy_postprocessor.json b/all_checkpoints/070000/pretrained_model/policy_postprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..0e896c35807a4a0616f52b341799a83165d6380e --- /dev/null +++ b/all_checkpoints/070000/pretrained_model/policy_postprocessor.json @@ -0,0 +1,32 @@ +{ + "name": "policy_postprocessor", + "steps": [ + { + "registry_name": "unnormalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "norm_map": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + } + }, + "state_file": "policy_postprocessor_step_0_unnormalizer_processor.safetensors" + }, + { + "registry_name": "device_processor", + "config": { + "device": "cpu", + "float_dtype": null + } + } + ] +} \ No newline at end of file diff --git a/all_checkpoints/070000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors b/all_checkpoints/070000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..05a9ab66dd9832301075a056e668bf83ae70aca5 --- /dev/null +++ b/all_checkpoints/070000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebcf8788ac2c38ccae8bc139e57a83da88fa38f6ab01a8fcc40cdbccdeb5286b +size 7608 diff --git a/all_checkpoints/070000/pretrained_model/policy_preprocessor.json b/all_checkpoints/070000/pretrained_model/policy_preprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..3254661686f66abc7b9baef1389d4ee961fcd9df --- /dev/null +++ b/all_checkpoints/070000/pretrained_model/policy_preprocessor.json @@ -0,0 +1,75 @@ +{ + "name": "policy_preprocessor", + "steps": [ + { + "registry_name": "rename_observations_processor", + "config": { + "rename_map": {} + } + }, + { + "registry_name": "to_batch_processor", + "config": {} + }, + { + "registry_name": "tokenizer_processor", + "config": { + "max_length": 77, + "task_key": "task", + "padding_side": "right", + "padding": "max_length", + "truncation": true, + "tokenizer_name": "openai/clip-vit-base-patch16" + } + }, + { + "registry_name": "device_processor", + "config": { + "device": "cuda", + "float_dtype": null + } + }, + { + "registry_name": "normalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "observation.state": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "observation.images.cam1": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + }, + "observation.images.cam2": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + }, + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "norm_map": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + } + }, + "state_file": "policy_preprocessor_step_4_normalizer_processor.safetensors" + } + ] +} \ No newline at end of file diff --git a/all_checkpoints/070000/pretrained_model/policy_preprocessor_step_4_normalizer_processor.safetensors b/all_checkpoints/070000/pretrained_model/policy_preprocessor_step_4_normalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..941aac9535d2ebd0a2779666c91e761ed89493e6 --- /dev/null +++ b/all_checkpoints/070000/pretrained_model/policy_preprocessor_step_4_normalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1483261062d5f6b8a80141ebb343170a30f06f9cda00c5b433b378d54610753c +size 7616 diff --git a/all_checkpoints/070000/pretrained_model/train_config.json b/all_checkpoints/070000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5d48afc7f96c2a80c152eba766f71b4dcd138706 --- /dev/null +++ b/all_checkpoints/070000/pretrained_model/train_config.json @@ -0,0 +1,257 @@ +{ + "dataset": { + "repo_id": "Bigenlight/cube_in_cup_lerobot_v3", + "root": "/home/junhyeong/workspace/youngwoong/cube_flow_matching/dataset", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + }, + "affine": { + "weight": 1.0, + "type": "RandomAffine", + "kwargs": { + "degrees": [ + -5.0, + 5.0 + ], + "translate": [ + 0.05, + 0.05 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec", + "return_uint8": false, + "depth_output_unit": "mm", + "streaming": false, + "eval_split": 0.13 + }, + "env": null, + "policy": { + "type": "multi_task_dit", + "n_obs_steps": 2, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "observation.images.cam1": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + }, + "observation.images.cam2": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "device": "cuda", + "use_amp": false, + "use_peft": false, + "push_to_hub": false, + "repo_id": null, + "private": null, + "tags": null, + "license": null, + "pretrained_path": null, + "pretrained_revision": null, + "horizon": 32, + "n_action_steps": 24, + "objective": "flow_matching", + "noise_scheduler_type": "DDPM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": null, + "sigma_min": 0.0, + "num_integration_steps": 100, + "integration_method": "euler", + "timestep_sampling_strategy": "beta", + "timestep_sampling_s": 0.999, + "timestep_sampling_alpha": 1.5, + "timestep_sampling_beta": 1.0, + "hidden_dim": 512, + "num_layers": 6, + "num_heads": 8, + "dropout": 0.1, + "use_positional_encoding": false, + "timestep_embed_dim": 256, + "use_rope": true, + "rope_base": 10000.0, + "vision_encoder_name": "openai/clip-vit-base-patch16", + "use_separate_rgb_encoder_per_camera": false, + "vision_encoder_lr_multiplier": 0.1, + "image_resize_shape": [ + 224, + 224 + ], + "image_crop_shape": [ + 224, + 224 + ], + "image_crop_is_random": true, + "text_encoder_name": "openai/clip-vit-base-patch16", + "tokenizer_max_length": 77, + "tokenizer_padding": "max_length", + "tokenizer_padding_side": "right", + "tokenizer_truncation": true, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "optimizer_lr": 2e-05, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 0.0, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 0, + "do_mask_loss_for_padding": false, + "drop_n_last_frames": 7 + }, + "reward_model": null, + "output_dir": "/home/junhyeong/workspace/youngwoong/cube_flow_matching/training/outputs/flow_matching_cube_in_cup", + "job_name": "flow_matching_cube_in_cup", + "resume": false, + "seed": 1000, + "cudnn_deterministic": false, + "num_workers": 4, + "batch_size": 8, + "prefetch_factor": 4, + "persistent_workers": true, + "steps": 100000, + "env_eval_freq": 20000, + "log_freq": 200, + "eval_steps": 2000, + "max_eval_samples": 0, + "tolerance_s": 0.0001, + "save_checkpoint": true, + "save_freq": 10000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 2e-05, + "weight_decay": 0.0, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 0, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 44, + "use_async_envs": true, + "recording": false, + "recording_repo_id": null, + "recording_private": false + }, + "wandb": { + "enable": false, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null, + "add_tags": true + }, + "peft": null, + "job": { + "target": null, + "image": "huggingface/lerobot-gpu:latest", + "timeout": "2d", + "detach": false, + "tags": [] + }, + "save_checkpoint_to_hub": false, + "sample_weighting": null, + "rename_map": {}, + "checkpoint_path": null +} \ No newline at end of file diff --git a/all_checkpoints/080000/pretrained_model/config.json b/all_checkpoints/080000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..a3874265d1aff2ae2f35bf5c14af146414f9f125 --- /dev/null +++ b/all_checkpoints/080000/pretrained_model/config.json @@ -0,0 +1,106 @@ +{ + "type": "multi_task_dit", + "n_obs_steps": 2, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "observation.images.cam1": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + }, + "observation.images.cam2": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "device": "cuda", + "use_amp": false, + "use_peft": false, + "push_to_hub": false, + "repo_id": null, + "private": null, + "tags": null, + "license": null, + "pretrained_path": null, + "pretrained_revision": null, + "horizon": 32, + "n_action_steps": 24, + "objective": "flow_matching", + "noise_scheduler_type": "DDPM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": null, + "sigma_min": 0.0, + "num_integration_steps": 100, + "integration_method": "euler", + "timestep_sampling_strategy": "beta", + "timestep_sampling_s": 0.999, + "timestep_sampling_alpha": 1.5, + "timestep_sampling_beta": 1.0, + "hidden_dim": 512, + "num_layers": 6, + "num_heads": 8, + "dropout": 0.1, + "use_positional_encoding": false, + "timestep_embed_dim": 256, + "use_rope": true, + "rope_base": 10000.0, + "vision_encoder_name": "openai/clip-vit-base-patch16", + "use_separate_rgb_encoder_per_camera": false, + "vision_encoder_lr_multiplier": 0.1, + "image_resize_shape": [ + 224, + 224 + ], + "image_crop_shape": [ + 224, + 224 + ], + "image_crop_is_random": true, + "text_encoder_name": "openai/clip-vit-base-patch16", + "tokenizer_max_length": 77, + "tokenizer_padding": "max_length", + "tokenizer_padding_side": "right", + "tokenizer_truncation": true, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "optimizer_lr": 2e-05, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 0.0, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 0, + "do_mask_loss_for_padding": false, + "drop_n_last_frames": 7 +} \ No newline at end of file diff --git a/all_checkpoints/080000/pretrained_model/model.safetensors b/all_checkpoints/080000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3fbeca5ec9bb9fe0acdfdc6ca563908bc50e24c1 --- /dev/null +++ b/all_checkpoints/080000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b2f49f4ae11385e1a0a01117e11d58dabd8823475e4895b5e4a0f29d55c9915 +size 995640924 diff --git a/all_checkpoints/080000/pretrained_model/policy_postprocessor.json b/all_checkpoints/080000/pretrained_model/policy_postprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..0e896c35807a4a0616f52b341799a83165d6380e --- /dev/null +++ b/all_checkpoints/080000/pretrained_model/policy_postprocessor.json @@ -0,0 +1,32 @@ +{ + "name": "policy_postprocessor", + "steps": [ + { + "registry_name": "unnormalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "norm_map": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + } + }, + "state_file": "policy_postprocessor_step_0_unnormalizer_processor.safetensors" + }, + { + "registry_name": "device_processor", + "config": { + "device": "cpu", + "float_dtype": null + } + } + ] +} \ No newline at end of file diff --git a/all_checkpoints/080000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors b/all_checkpoints/080000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..05a9ab66dd9832301075a056e668bf83ae70aca5 --- /dev/null +++ b/all_checkpoints/080000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebcf8788ac2c38ccae8bc139e57a83da88fa38f6ab01a8fcc40cdbccdeb5286b +size 7608 diff --git a/all_checkpoints/080000/pretrained_model/policy_preprocessor.json b/all_checkpoints/080000/pretrained_model/policy_preprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..3254661686f66abc7b9baef1389d4ee961fcd9df --- /dev/null +++ b/all_checkpoints/080000/pretrained_model/policy_preprocessor.json @@ -0,0 +1,75 @@ +{ + "name": "policy_preprocessor", + "steps": [ + { + "registry_name": "rename_observations_processor", + "config": { + "rename_map": {} + } + }, + { + "registry_name": "to_batch_processor", + "config": {} + }, + { + "registry_name": "tokenizer_processor", + "config": { + "max_length": 77, + "task_key": "task", + "padding_side": "right", + "padding": "max_length", + "truncation": true, + "tokenizer_name": "openai/clip-vit-base-patch16" + } + }, + { + "registry_name": "device_processor", + "config": { + "device": "cuda", + "float_dtype": null + } + }, + { + "registry_name": "normalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "observation.state": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "observation.images.cam1": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + }, + "observation.images.cam2": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + }, + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "norm_map": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + } + }, + "state_file": "policy_preprocessor_step_4_normalizer_processor.safetensors" + } + ] +} \ No newline at end of file diff --git a/all_checkpoints/080000/pretrained_model/policy_preprocessor_step_4_normalizer_processor.safetensors b/all_checkpoints/080000/pretrained_model/policy_preprocessor_step_4_normalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..941aac9535d2ebd0a2779666c91e761ed89493e6 --- /dev/null +++ b/all_checkpoints/080000/pretrained_model/policy_preprocessor_step_4_normalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1483261062d5f6b8a80141ebb343170a30f06f9cda00c5b433b378d54610753c +size 7616 diff --git a/all_checkpoints/080000/pretrained_model/train_config.json b/all_checkpoints/080000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5d48afc7f96c2a80c152eba766f71b4dcd138706 --- /dev/null +++ b/all_checkpoints/080000/pretrained_model/train_config.json @@ -0,0 +1,257 @@ +{ + "dataset": { + "repo_id": "Bigenlight/cube_in_cup_lerobot_v3", + "root": "/home/junhyeong/workspace/youngwoong/cube_flow_matching/dataset", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + }, + "affine": { + "weight": 1.0, + "type": "RandomAffine", + "kwargs": { + "degrees": [ + -5.0, + 5.0 + ], + "translate": [ + 0.05, + 0.05 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec", + "return_uint8": false, + "depth_output_unit": "mm", + "streaming": false, + "eval_split": 0.13 + }, + "env": null, + "policy": { + "type": "multi_task_dit", + "n_obs_steps": 2, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "observation.images.cam1": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + }, + "observation.images.cam2": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "device": "cuda", + "use_amp": false, + "use_peft": false, + "push_to_hub": false, + "repo_id": null, + "private": null, + "tags": null, + "license": null, + "pretrained_path": null, + "pretrained_revision": null, + "horizon": 32, + "n_action_steps": 24, + "objective": "flow_matching", + "noise_scheduler_type": "DDPM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": null, + "sigma_min": 0.0, + "num_integration_steps": 100, + "integration_method": "euler", + "timestep_sampling_strategy": "beta", + "timestep_sampling_s": 0.999, + "timestep_sampling_alpha": 1.5, + "timestep_sampling_beta": 1.0, + "hidden_dim": 512, + "num_layers": 6, + "num_heads": 8, + "dropout": 0.1, + "use_positional_encoding": false, + "timestep_embed_dim": 256, + "use_rope": true, + "rope_base": 10000.0, + "vision_encoder_name": "openai/clip-vit-base-patch16", + "use_separate_rgb_encoder_per_camera": false, + "vision_encoder_lr_multiplier": 0.1, + "image_resize_shape": [ + 224, + 224 + ], + "image_crop_shape": [ + 224, + 224 + ], + "image_crop_is_random": true, + "text_encoder_name": "openai/clip-vit-base-patch16", + "tokenizer_max_length": 77, + "tokenizer_padding": "max_length", + "tokenizer_padding_side": "right", + "tokenizer_truncation": true, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "optimizer_lr": 2e-05, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 0.0, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 0, + "do_mask_loss_for_padding": false, + "drop_n_last_frames": 7 + }, + "reward_model": null, + "output_dir": "/home/junhyeong/workspace/youngwoong/cube_flow_matching/training/outputs/flow_matching_cube_in_cup", + "job_name": "flow_matching_cube_in_cup", + "resume": false, + "seed": 1000, + "cudnn_deterministic": false, + "num_workers": 4, + "batch_size": 8, + "prefetch_factor": 4, + "persistent_workers": true, + "steps": 100000, + "env_eval_freq": 20000, + "log_freq": 200, + "eval_steps": 2000, + "max_eval_samples": 0, + "tolerance_s": 0.0001, + "save_checkpoint": true, + "save_freq": 10000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 2e-05, + "weight_decay": 0.0, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 0, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 44, + "use_async_envs": true, + "recording": false, + "recording_repo_id": null, + "recording_private": false + }, + "wandb": { + "enable": false, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null, + "add_tags": true + }, + "peft": null, + "job": { + "target": null, + "image": "huggingface/lerobot-gpu:latest", + "timeout": "2d", + "detach": false, + "tags": [] + }, + "save_checkpoint_to_hub": false, + "sample_weighting": null, + "rename_map": {}, + "checkpoint_path": null +} \ No newline at end of file diff --git a/all_checkpoints/090000/pretrained_model/config.json b/all_checkpoints/090000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..a3874265d1aff2ae2f35bf5c14af146414f9f125 --- /dev/null +++ b/all_checkpoints/090000/pretrained_model/config.json @@ -0,0 +1,106 @@ +{ + "type": "multi_task_dit", + "n_obs_steps": 2, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "observation.images.cam1": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + }, + "observation.images.cam2": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "device": "cuda", + "use_amp": false, + "use_peft": false, + "push_to_hub": false, + "repo_id": null, + "private": null, + "tags": null, + "license": null, + "pretrained_path": null, + "pretrained_revision": null, + "horizon": 32, + "n_action_steps": 24, + "objective": "flow_matching", + "noise_scheduler_type": "DDPM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": null, + "sigma_min": 0.0, + "num_integration_steps": 100, + "integration_method": "euler", + "timestep_sampling_strategy": "beta", + "timestep_sampling_s": 0.999, + "timestep_sampling_alpha": 1.5, + "timestep_sampling_beta": 1.0, + "hidden_dim": 512, + "num_layers": 6, + "num_heads": 8, + "dropout": 0.1, + "use_positional_encoding": false, + "timestep_embed_dim": 256, + "use_rope": true, + "rope_base": 10000.0, + "vision_encoder_name": "openai/clip-vit-base-patch16", + "use_separate_rgb_encoder_per_camera": false, + "vision_encoder_lr_multiplier": 0.1, + "image_resize_shape": [ + 224, + 224 + ], + "image_crop_shape": [ + 224, + 224 + ], + "image_crop_is_random": true, + "text_encoder_name": "openai/clip-vit-base-patch16", + "tokenizer_max_length": 77, + "tokenizer_padding": "max_length", + "tokenizer_padding_side": "right", + "tokenizer_truncation": true, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "optimizer_lr": 2e-05, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 0.0, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 0, + "do_mask_loss_for_padding": false, + "drop_n_last_frames": 7 +} \ No newline at end of file diff --git a/all_checkpoints/090000/pretrained_model/model.safetensors b/all_checkpoints/090000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..614e6f8fb3e02d2a3683093a0858b460f0885891 --- /dev/null +++ b/all_checkpoints/090000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab1c7297abdc47c7234d48fa490d609d4b66ba73c4a760a073427b1ec74c5a48 +size 995640924 diff --git a/all_checkpoints/090000/pretrained_model/policy_postprocessor.json b/all_checkpoints/090000/pretrained_model/policy_postprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..0e896c35807a4a0616f52b341799a83165d6380e --- /dev/null +++ b/all_checkpoints/090000/pretrained_model/policy_postprocessor.json @@ -0,0 +1,32 @@ +{ + "name": "policy_postprocessor", + "steps": [ + { + "registry_name": "unnormalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "norm_map": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + } + }, + "state_file": "policy_postprocessor_step_0_unnormalizer_processor.safetensors" + }, + { + "registry_name": "device_processor", + "config": { + "device": "cpu", + "float_dtype": null + } + } + ] +} \ No newline at end of file diff --git a/all_checkpoints/090000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors b/all_checkpoints/090000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..05a9ab66dd9832301075a056e668bf83ae70aca5 --- /dev/null +++ b/all_checkpoints/090000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebcf8788ac2c38ccae8bc139e57a83da88fa38f6ab01a8fcc40cdbccdeb5286b +size 7608 diff --git a/all_checkpoints/090000/pretrained_model/policy_preprocessor.json b/all_checkpoints/090000/pretrained_model/policy_preprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..3254661686f66abc7b9baef1389d4ee961fcd9df --- /dev/null +++ b/all_checkpoints/090000/pretrained_model/policy_preprocessor.json @@ -0,0 +1,75 @@ +{ + "name": "policy_preprocessor", + "steps": [ + { + "registry_name": "rename_observations_processor", + "config": { + "rename_map": {} + } + }, + { + "registry_name": "to_batch_processor", + "config": {} + }, + { + "registry_name": "tokenizer_processor", + "config": { + "max_length": 77, + "task_key": "task", + "padding_side": "right", + "padding": "max_length", + "truncation": true, + "tokenizer_name": "openai/clip-vit-base-patch16" + } + }, + { + "registry_name": "device_processor", + "config": { + "device": "cuda", + "float_dtype": null + } + }, + { + "registry_name": "normalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "observation.state": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "observation.images.cam1": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + }, + "observation.images.cam2": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + }, + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "norm_map": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + } + }, + "state_file": "policy_preprocessor_step_4_normalizer_processor.safetensors" + } + ] +} \ No newline at end of file diff --git a/all_checkpoints/090000/pretrained_model/policy_preprocessor_step_4_normalizer_processor.safetensors b/all_checkpoints/090000/pretrained_model/policy_preprocessor_step_4_normalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..941aac9535d2ebd0a2779666c91e761ed89493e6 --- /dev/null +++ b/all_checkpoints/090000/pretrained_model/policy_preprocessor_step_4_normalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1483261062d5f6b8a80141ebb343170a30f06f9cda00c5b433b378d54610753c +size 7616 diff --git a/all_checkpoints/090000/pretrained_model/train_config.json b/all_checkpoints/090000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5d48afc7f96c2a80c152eba766f71b4dcd138706 --- /dev/null +++ b/all_checkpoints/090000/pretrained_model/train_config.json @@ -0,0 +1,257 @@ +{ + "dataset": { + "repo_id": "Bigenlight/cube_in_cup_lerobot_v3", + "root": "/home/junhyeong/workspace/youngwoong/cube_flow_matching/dataset", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + }, + "affine": { + "weight": 1.0, + "type": "RandomAffine", + "kwargs": { + "degrees": [ + -5.0, + 5.0 + ], + "translate": [ + 0.05, + 0.05 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec", + "return_uint8": false, + "depth_output_unit": "mm", + "streaming": false, + "eval_split": 0.13 + }, + "env": null, + "policy": { + "type": "multi_task_dit", + "n_obs_steps": 2, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "observation.images.cam1": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + }, + "observation.images.cam2": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "device": "cuda", + "use_amp": false, + "use_peft": false, + "push_to_hub": false, + "repo_id": null, + "private": null, + "tags": null, + "license": null, + "pretrained_path": null, + "pretrained_revision": null, + "horizon": 32, + "n_action_steps": 24, + "objective": "flow_matching", + "noise_scheduler_type": "DDPM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": null, + "sigma_min": 0.0, + "num_integration_steps": 100, + "integration_method": "euler", + "timestep_sampling_strategy": "beta", + "timestep_sampling_s": 0.999, + "timestep_sampling_alpha": 1.5, + "timestep_sampling_beta": 1.0, + "hidden_dim": 512, + "num_layers": 6, + "num_heads": 8, + "dropout": 0.1, + "use_positional_encoding": false, + "timestep_embed_dim": 256, + "use_rope": true, + "rope_base": 10000.0, + "vision_encoder_name": "openai/clip-vit-base-patch16", + "use_separate_rgb_encoder_per_camera": false, + "vision_encoder_lr_multiplier": 0.1, + "image_resize_shape": [ + 224, + 224 + ], + "image_crop_shape": [ + 224, + 224 + ], + "image_crop_is_random": true, + "text_encoder_name": "openai/clip-vit-base-patch16", + "tokenizer_max_length": 77, + "tokenizer_padding": "max_length", + "tokenizer_padding_side": "right", + "tokenizer_truncation": true, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "optimizer_lr": 2e-05, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 0.0, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 0, + "do_mask_loss_for_padding": false, + "drop_n_last_frames": 7 + }, + "reward_model": null, + "output_dir": "/home/junhyeong/workspace/youngwoong/cube_flow_matching/training/outputs/flow_matching_cube_in_cup", + "job_name": "flow_matching_cube_in_cup", + "resume": false, + "seed": 1000, + "cudnn_deterministic": false, + "num_workers": 4, + "batch_size": 8, + "prefetch_factor": 4, + "persistent_workers": true, + "steps": 100000, + "env_eval_freq": 20000, + "log_freq": 200, + "eval_steps": 2000, + "max_eval_samples": 0, + "tolerance_s": 0.0001, + "save_checkpoint": true, + "save_freq": 10000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 2e-05, + "weight_decay": 0.0, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 0, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 44, + "use_async_envs": true, + "recording": false, + "recording_repo_id": null, + "recording_private": false + }, + "wandb": { + "enable": false, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null, + "add_tags": true + }, + "peft": null, + "job": { + "target": null, + "image": "huggingface/lerobot-gpu:latest", + "timeout": "2d", + "detach": false, + "tags": [] + }, + "save_checkpoint_to_hub": false, + "sample_weighting": null, + "rename_map": {}, + "checkpoint_path": null +} \ No newline at end of file diff --git a/all_checkpoints/100000/pretrained_model/config.json b/all_checkpoints/100000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..a3874265d1aff2ae2f35bf5c14af146414f9f125 --- /dev/null +++ b/all_checkpoints/100000/pretrained_model/config.json @@ -0,0 +1,106 @@ +{ + "type": "multi_task_dit", + "n_obs_steps": 2, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "observation.images.cam1": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + }, + "observation.images.cam2": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "device": "cuda", + "use_amp": false, + "use_peft": false, + "push_to_hub": false, + "repo_id": null, + "private": null, + "tags": null, + "license": null, + "pretrained_path": null, + "pretrained_revision": null, + "horizon": 32, + "n_action_steps": 24, + "objective": "flow_matching", + "noise_scheduler_type": "DDPM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": null, + "sigma_min": 0.0, + "num_integration_steps": 100, + "integration_method": "euler", + "timestep_sampling_strategy": "beta", + "timestep_sampling_s": 0.999, + "timestep_sampling_alpha": 1.5, + "timestep_sampling_beta": 1.0, + "hidden_dim": 512, + "num_layers": 6, + "num_heads": 8, + "dropout": 0.1, + "use_positional_encoding": false, + "timestep_embed_dim": 256, + "use_rope": true, + "rope_base": 10000.0, + "vision_encoder_name": "openai/clip-vit-base-patch16", + "use_separate_rgb_encoder_per_camera": false, + "vision_encoder_lr_multiplier": 0.1, + "image_resize_shape": [ + 224, + 224 + ], + "image_crop_shape": [ + 224, + 224 + ], + "image_crop_is_random": true, + "text_encoder_name": "openai/clip-vit-base-patch16", + "tokenizer_max_length": 77, + "tokenizer_padding": "max_length", + "tokenizer_padding_side": "right", + "tokenizer_truncation": true, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "optimizer_lr": 2e-05, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 0.0, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 0, + "do_mask_loss_for_padding": false, + "drop_n_last_frames": 7 +} \ No newline at end of file diff --git a/all_checkpoints/100000/pretrained_model/model.safetensors b/all_checkpoints/100000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..88a64cd3b1de852812092fc34644b2b24dcedb90 --- /dev/null +++ b/all_checkpoints/100000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58be42703fb429732ef5be35cbd25cebd7717392d1138ac08e4ad95dff4bd947 +size 995640924 diff --git a/all_checkpoints/100000/pretrained_model/policy_postprocessor.json b/all_checkpoints/100000/pretrained_model/policy_postprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..0e896c35807a4a0616f52b341799a83165d6380e --- /dev/null +++ b/all_checkpoints/100000/pretrained_model/policy_postprocessor.json @@ -0,0 +1,32 @@ +{ + "name": "policy_postprocessor", + "steps": [ + { + "registry_name": "unnormalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "norm_map": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + } + }, + "state_file": "policy_postprocessor_step_0_unnormalizer_processor.safetensors" + }, + { + "registry_name": "device_processor", + "config": { + "device": "cpu", + "float_dtype": null + } + } + ] +} \ No newline at end of file diff --git a/all_checkpoints/100000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors b/all_checkpoints/100000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..05a9ab66dd9832301075a056e668bf83ae70aca5 --- /dev/null +++ b/all_checkpoints/100000/pretrained_model/policy_postprocessor_step_0_unnormalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebcf8788ac2c38ccae8bc139e57a83da88fa38f6ab01a8fcc40cdbccdeb5286b +size 7608 diff --git a/all_checkpoints/100000/pretrained_model/policy_preprocessor.json b/all_checkpoints/100000/pretrained_model/policy_preprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..3254661686f66abc7b9baef1389d4ee961fcd9df --- /dev/null +++ b/all_checkpoints/100000/pretrained_model/policy_preprocessor.json @@ -0,0 +1,75 @@ +{ + "name": "policy_preprocessor", + "steps": [ + { + "registry_name": "rename_observations_processor", + "config": { + "rename_map": {} + } + }, + { + "registry_name": "to_batch_processor", + "config": {} + }, + { + "registry_name": "tokenizer_processor", + "config": { + "max_length": 77, + "task_key": "task", + "padding_side": "right", + "padding": "max_length", + "truncation": true, + "tokenizer_name": "openai/clip-vit-base-patch16" + } + }, + { + "registry_name": "device_processor", + "config": { + "device": "cuda", + "float_dtype": null + } + }, + { + "registry_name": "normalizer_processor", + "config": { + "eps": 1e-08, + "features": { + "observation.state": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "observation.images.cam1": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + }, + "observation.images.cam2": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + }, + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "norm_map": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + } + }, + "state_file": "policy_preprocessor_step_4_normalizer_processor.safetensors" + } + ] +} \ No newline at end of file diff --git a/all_checkpoints/100000/pretrained_model/policy_preprocessor_step_4_normalizer_processor.safetensors b/all_checkpoints/100000/pretrained_model/policy_preprocessor_step_4_normalizer_processor.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..941aac9535d2ebd0a2779666c91e761ed89493e6 --- /dev/null +++ b/all_checkpoints/100000/pretrained_model/policy_preprocessor_step_4_normalizer_processor.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1483261062d5f6b8a80141ebb343170a30f06f9cda00c5b433b378d54610753c +size 7616 diff --git a/all_checkpoints/100000/pretrained_model/train_config.json b/all_checkpoints/100000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5d48afc7f96c2a80c152eba766f71b4dcd138706 --- /dev/null +++ b/all_checkpoints/100000/pretrained_model/train_config.json @@ -0,0 +1,257 @@ +{ + "dataset": { + "repo_id": "Bigenlight/cube_in_cup_lerobot_v3", + "root": "/home/junhyeong/workspace/youngwoong/cube_flow_matching/dataset", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + }, + "affine": { + "weight": 1.0, + "type": "RandomAffine", + "kwargs": { + "degrees": [ + -5.0, + 5.0 + ], + "translate": [ + 0.05, + 0.05 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec", + "return_uint8": false, + "depth_output_unit": "mm", + "streaming": false, + "eval_split": 0.13 + }, + "env": null, + "policy": { + "type": "multi_task_dit", + "n_obs_steps": 2, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 7 + ] + }, + "observation.images.cam1": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + }, + "observation.images.cam2": { + "type": "VISUAL", + "shape": [ + 3, + 720, + 1280 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 7 + ] + } + }, + "device": "cuda", + "use_amp": false, + "use_peft": false, + "push_to_hub": false, + "repo_id": null, + "private": null, + "tags": null, + "license": null, + "pretrained_path": null, + "pretrained_revision": null, + "horizon": 32, + "n_action_steps": 24, + "objective": "flow_matching", + "noise_scheduler_type": "DDPM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": null, + "sigma_min": 0.0, + "num_integration_steps": 100, + "integration_method": "euler", + "timestep_sampling_strategy": "beta", + "timestep_sampling_s": 0.999, + "timestep_sampling_alpha": 1.5, + "timestep_sampling_beta": 1.0, + "hidden_dim": 512, + "num_layers": 6, + "num_heads": 8, + "dropout": 0.1, + "use_positional_encoding": false, + "timestep_embed_dim": 256, + "use_rope": true, + "rope_base": 10000.0, + "vision_encoder_name": "openai/clip-vit-base-patch16", + "use_separate_rgb_encoder_per_camera": false, + "vision_encoder_lr_multiplier": 0.1, + "image_resize_shape": [ + 224, + 224 + ], + "image_crop_shape": [ + 224, + 224 + ], + "image_crop_is_random": true, + "text_encoder_name": "openai/clip-vit-base-patch16", + "tokenizer_max_length": 77, + "tokenizer_padding": "max_length", + "tokenizer_padding_side": "right", + "tokenizer_truncation": true, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "optimizer_lr": 2e-05, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 0.0, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 0, + "do_mask_loss_for_padding": false, + "drop_n_last_frames": 7 + }, + "reward_model": null, + "output_dir": "/home/junhyeong/workspace/youngwoong/cube_flow_matching/training/outputs/flow_matching_cube_in_cup", + "job_name": "flow_matching_cube_in_cup", + "resume": false, + "seed": 1000, + "cudnn_deterministic": false, + "num_workers": 4, + "batch_size": 8, + "prefetch_factor": 4, + "persistent_workers": true, + "steps": 100000, + "env_eval_freq": 20000, + "log_freq": 200, + "eval_steps": 2000, + "max_eval_samples": 0, + "tolerance_s": 0.0001, + "save_checkpoint": true, + "save_freq": 10000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 2e-05, + "weight_decay": 0.0, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 0, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 44, + "use_async_envs": true, + "recording": false, + "recording_repo_id": null, + "recording_private": false + }, + "wandb": { + "enable": false, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null, + "add_tags": true + }, + "peft": null, + "job": { + "target": null, + "image": "huggingface/lerobot-gpu:latest", + "timeout": "2d", + "detach": false, + "tags": [] + }, + "save_checkpoint_to_hub": false, + "sample_weighting": null, + "rename_map": {}, + "checkpoint_path": null +} \ No newline at end of file