final checkpoint (114.4 epochs) + model card

Browse files

Files changed (8) hide show

README.md +88 -0
config.json +90 -0
model.safetensors +3 -0
policy_postprocessor.json +33 -0
policy_postprocessor_step_0_unnormalizer_processor.safetensors +3 -0
policy_preprocessor.json +76 -0
policy_preprocessor_step_4_normalizer_processor.safetensors +3 -0
train_config.json +243 -0

README.md ADDED Viewed

	@@ -0,0 +1,88 @@

+---
+license: apache-2.0
+library_name: lerobot
+tags: [robotics, lerobot, act, tactile, manipulation]
+datasets: [aryankakad/tactile_charger_inserting]
+---
+# act_tactile_charger_20ep
+**ACT** policy — **vision + tactile** — trained on **20 episodes**.
+Task: *grab and remove the charger from the socket and put it in the black box*
+Trained with [LeFlexiTac](https://github.com/TNA001-AI/lerobot_tactile), a LeRobot fork adding
+FlexiTac tactile sensing. Docs: <https://tna001-ai.github.io/LeFlexiTac/docs.html>
+## Training data
+| | |
+|---|---|
+| dataset | [`aryankakad/tactile_charger_inserting`](https://huggingface.co/datasets/aryankakad/tactile_charger_inserting) |
+| episodes | 20 (indices 0–19) |
+| frames | 10,717 @ 30 fps |
+| cameras | `observation.images.top`, `observation.images.gripper` (224×224) |
+| tactile | `observation.tactile.primary` (12×32) |
+| state / action | 6-DoF SO-100 follower |
+## Configuration
+| | |
+|---|---|
+| steps | 38,300 |
+| batch size | 32 |
+| **epochs** | **114.4** |
+| `chunk_size` | `100` |
+| `n_action_steps` | `100` |
+| `vision_backbone` | `resnet18` |
+| `dim_model` | `512` |
+| `n_encoder_layers` | `4` |
+| `n_decoder_layers` | `1` |
+| `use_vae` | `True` |
+| `kl_weight` | `10.0` |
+| `optimizer_lr` | `1e-05` |
+| `optimizer_weight_decay` | `0.0001` |
+| `n_tactile_tokens` | `4` |
+| `tactile_encoder_type` | `cnn` |
+Every model in this series is **epoch-matched at ~114.4 epochs**, so dataset size and
+sensor modality are the only variables across the set.
+## Training command actually used
+Run on 1× AMD Instinct MI300X (ROCm 6.2.4). `HIP_VISIBLE_DEVICES` selected the GPU,
+so `--policy.device=cuda` refers to that single card.
+```bash
+python -u -m lerobot.scripts.lerobot_train \
+  --dataset.repo_id=aryankakad/tactile_charger_inserting \
+  --dataset.episodes='[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]' \
+  --policy.type=act --policy.use_tactile=true \
+  --policy.tactile_features='["observation.tactile.primary"]' \
+  --policy.n_tactile_tokens=4 \
+  --policy.repo_id=Dimios45/act_tactile_charger_20ep \
+  --policy.private=true --policy.device=cuda \
+  --output_dir=outputs/train/B_act_tactile_20ep --job_name=B_act_tactile_20ep \
+  --batch_size=32 --num_workers=8 --steps=38300 --save_freq=10000 --wandb.enable=true
+```
+## Evaluation / rollout
+Not run here — this machine has no robot attached. To evaluate, run on the machine with the
+SO-100 and sensors, loading the policy with `--policy.path=Dimios45/act_tactile_charger_20ep`.
+Reference: the `lerobot-record` eval invocations in
+[`tactile_cmd.txt`](https://github.com/TNA001-AI/lerobot_tactile/blob/main/tactile_cmd.txt)
+and the [project docs](https://tna001-ai.github.io/LeFlexiTac/docs.html). You will need to
+supply your own robot port, camera serials, and a `primary` tactile sensor entry matching training.
+## Notes
+- Two ROCm-specific fixes were required in the fork: `persistent_workers=True` on the
+  dataloader (epoch boundaries otherwise stalled ~410 s each), and keeping
+  `cudnn.benchmark` **off** (on ROCm it triggers an exhaustive MIOpen search that can
+  precede step 1 by hours).
+- Training loss is **not** a proxy for task success. Compare policies by rollout success
+  rate, especially on contact-rich phases.
+- The source dataset's task string is labelled `stack cup` — a mislabel carried over from
+  an earlier session. It does not affect ACT, which is not language-conditioned.

config.json ADDED Viewed

	@@ -0,0 +1,90 @@

+{
+    "type": "act",
+    "n_obs_steps": 1,
+    "input_features": {
+        "observation.state": {
+            "type": "STATE",
+            "shape": [
+                6
+            ]
+        },
+        "observation.images.gripper": {
+            "type": "VISUAL",
+            "shape": [
+                3,
+                224,
+                224
+            ]
+        },
+        "observation.images.top": {
+            "type": "VISUAL",
+            "shape": [
+                3,
+                224,
+                224
+            ]
+        },
+        "observation.tactile.primary": {
+            "type": "STATE",
+            "shape": [
+                12,
+                32
+            ]
+        }
+    },
+    "output_features": {
+        "action": {
+            "type": "ACTION",
+            "shape": [
+                6
+            ]
+        }
+    },
+    "device": "cuda",
+    "use_amp": false,
+    "use_peft": false,
+    "push_to_hub": true,
+    "repo_id": "Dimios45/act_tactile_charger_20ep",
+    "private": true,
+    "tags": null,
+    "license": null,
+    "pretrained_path": null,
+    "chunk_size": 100,
+    "n_action_steps": 100,
+    "normalization_mapping": {
+        "VISUAL": "MEAN_STD",
+        "STATE": "MEAN_STD",
+        "ACTION": "MEAN_STD",
+        "TACTILE": "MEAN_STD"
+    },
+    "vision_backbone": "resnet18",
+    "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1",
+    "replace_final_stride_with_dilation": false,
+    "pre_norm": false,
+    "dim_model": 512,
+    "n_heads": 8,
+    "dim_feedforward": 3200,
+    "feedforward_activation": "relu",
+    "n_encoder_layers": 4,
+    "n_decoder_layers": 1,
+    "use_vae": true,
+    "latent_dim": 32,
+    "n_vae_encoder_layers": 4,
+    "temporal_ensemble_coeff": null,
+    "use_tactile": true,
+    "tactile_encoder_type": "cnn",
+    "tactile_input_shape": [
+        12,
+        32
+    ],
+    "tactile_dropout": 0.3,
+    "tactile_features": [
+        "observation.tactile.primary"
+    ],
+    "n_tactile_tokens": 4,
+    "dropout": 0.1,
+    "kl_weight": 10.0,
+    "optimizer_lr": 1e-05,
+    "optimizer_weight_decay": 0.0001,
+    "optimizer_lr_backbone": 1e-05
+}

model.safetensors ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:0e011a2e00263c8e2cc30ce50b64c39865eebf11aa1a6760d452933accf96621
+size 213388992

policy_postprocessor.json ADDED Viewed

	@@ -0,0 +1,33 @@

+{
+  "name": "policy_postprocessor",
+  "steps": [
+    {
+      "registry_name": "unnormalizer_processor",
+      "config": {
+        "eps": 1e-08,
+        "features": {
+          "action": {
+            "type": "ACTION",
+            "shape": [
+              6
+            ]
+          }
+        },
+        "norm_map": {
+          "VISUAL": "MEAN_STD",
+          "STATE": "MEAN_STD",
+          "ACTION": "MEAN_STD",
+          "TACTILE": "MEAN_STD"
+        }
+      },
+      "state_file": "policy_postprocessor_step_0_unnormalizer_processor.safetensors"
+    },
+    {
+      "registry_name": "device_processor",
+      "config": {
+        "device": "cpu",
+        "float_dtype": null
+      }
+    }
+  ]
+}

policy_postprocessor_step_0_unnormalizer_processor.safetensors ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:44d4d7be9139b14b3b0810abf88febcbcc7f3862b89e7f2783278919aab3e232
+size 9644

policy_preprocessor.json ADDED Viewed

	@@ -0,0 +1,76 @@

+{
+  "name": "policy_preprocessor",
+  "steps": [
+    {
+      "registry_name": "rename_observations_processor",
+      "config": {
+        "rename_map": {}
+      }
+    },
+    {
+      "class": "lerobot.processor.tactile_processor.TactileValidationProcessorStep",
+      "config": {}
+    },
+    {
+      "registry_name": "to_batch_processor",
+      "config": {}
+    },
+    {
+      "registry_name": "device_processor",
+      "config": {
+        "device": "cuda",
+        "float_dtype": null
+      }
+    },
+    {
+      "registry_name": "normalizer_processor",
+      "config": {
+        "eps": 1e-08,
+        "features": {
+          "observation.state": {
+            "type": "STATE",
+            "shape": [
+              6
+            ]
+          },
+          "observation.images.gripper": {
+            "type": "VISUAL",
+            "shape": [
+              3,
+              224,
+              224
+            ]
+          },
+          "observation.images.top": {
+            "type": "VISUAL",
+            "shape": [
+              3,
+              224,
+              224
+            ]
+          },
+          "observation.tactile.primary": {
+            "type": "STATE",
+            "shape": [
+              12,
+              32
+            ]
+          },
+          "action": {
+            "type": "ACTION",
+            "shape": [
+              6
+            ]
+          }
+        },
+        "norm_map": {
+          "VISUAL": "MEAN_STD",
+          "STATE": "MEAN_STD",
+          "ACTION": "MEAN_STD",
+          "TACTILE": "MEAN_STD"
+        }
+      },
+      "state_file": "policy_preprocessor_step_4_normalizer_processor.safetensors"
+    }
+  ]
+}

policy_preprocessor_step_4_normalizer_processor.safetensors ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:44d4d7be9139b14b3b0810abf88febcbcc7f3862b89e7f2783278919aab3e232
+size 9644

train_config.json ADDED Viewed

	@@ -0,0 +1,243 @@

+{
+    "dataset": {
+        "repo_id": "aryankakad/tactile_charger_inserting",
+        "root": null,
+        "episodes": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15,
+            16,
+            17,
+            18,
+            19
+        ],
+        "image_transforms": {
+            "enable": false,
+            "max_num_transforms": 3,
+            "random_order": false,
+            "tfs": {
+                "brightness": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "brightness": [
+                            0.8,
+                            1.2
+                        ]
+                    }
+                },
+                "contrast": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "contrast": [
+                            0.8,
+                            1.2
+                        ]
+                    }
+                },
+                "saturation": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "saturation": [
+                            0.5,
+                            1.5
+                        ]
+                    }
+                },
+                "hue": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "hue": [
+                            -0.05,
+                            0.05
+                        ]
+                    }
+                },
+                "sharpness": {
+                    "weight": 1.0,
+                    "type": "SharpnessJitter",
+                    "kwargs": {
+                        "sharpness": [
+                            0.5,
+                            1.5
+                        ]
+                    }
+                },
+                "affine": {
+                    "weight": 1.0,
+                    "type": "RandomAffine",
+                    "kwargs": {
+                        "degrees": [
+                            -5.0,
+                            5.0
+                        ],
+                        "translate": [
+                            0.05,
+                            0.05
+                        ]
+                    }
+                }
+            }
+        },
+        "revision": null,
+        "use_imagenet_stats": true,
+        "video_backend": "torchcodec",
+        "streaming": false
+    },
+    "env": null,
+    "policy": {
+        "type": "act",
+        "n_obs_steps": 1,
+        "input_features": {
+            "observation.state": {
+                "type": "STATE",
+                "shape": [
+                    6
+                ]
+            },
+            "observation.images.gripper": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    224,
+                    224
+                ]
+            },
+            "observation.images.top": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    224,
+                    224
+                ]
+            },
+            "observation.tactile.primary": {
+                "type": "STATE",
+                "shape": [
+                    12,
+                    32
+                ]
+            }
+        },
+        "output_features": {
+            "action": {
+                "type": "ACTION",
+                "shape": [
+                    6
+                ]
+            }
+        },
+        "device": "cuda",
+        "use_amp": false,
+        "use_peft": false,
+        "push_to_hub": true,
+        "repo_id": "Dimios45/act_tactile_charger_20ep",
+        "private": true,
+        "tags": null,
+        "license": null,
+        "pretrained_path": null,
+        "chunk_size": 100,
+        "n_action_steps": 100,
+        "normalization_mapping": {
+            "VISUAL": "MEAN_STD",
+            "STATE": "MEAN_STD",
+            "ACTION": "MEAN_STD",
+            "TACTILE": "MEAN_STD"
+        },
+        "vision_backbone": "resnet18",
+        "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1",
+        "replace_final_stride_with_dilation": false,
+        "pre_norm": false,
+        "dim_model": 512,
+        "n_heads": 8,
+        "dim_feedforward": 3200,
+        "feedforward_activation": "relu",
+        "n_encoder_layers": 4,
+        "n_decoder_layers": 1,
+        "use_vae": true,
+        "latent_dim": 32,
+        "n_vae_encoder_layers": 4,
+        "temporal_ensemble_coeff": null,
+        "use_tactile": true,
+        "tactile_encoder_type": "cnn",
+        "tactile_input_shape": [
+            12,
+            32
+        ],
+        "tactile_dropout": 0.3,
+        "tactile_features": [
+            "observation.tactile.primary"
+        ],
+        "n_tactile_tokens": 4,
+        "dropout": 0.1,
+        "kl_weight": 10.0,
+        "optimizer_lr": 1e-05,
+        "optimizer_weight_decay": 0.0001,
+        "optimizer_lr_backbone": 1e-05
+    },
+    "output_dir": "outputs/train/B_act_tactile_20ep",
+    "job_name": "B_act_tactile_20ep",
+    "resume": false,
+    "seed": 1000,
+    "cudnn_deterministic": false,
+    "num_workers": 8,
+    "batch_size": 32,
+    "steps": 38300,
+    "eval_freq": 20000,
+    "log_freq": 200,
+    "tolerance_s": 0.0001,
+    "save_checkpoint": true,
+    "save_freq": 10000,
+    "use_policy_training_preset": true,
+    "optimizer": {
+        "type": "adamw",
+        "lr": 1e-05,
+        "weight_decay": 0.0001,
+        "grad_clip_norm": 10.0,
+        "betas": [
+            0.9,
+            0.999
+        ],
+        "eps": 1e-08
+    },
+    "scheduler": null,
+    "eval": {
+        "n_episodes": 50,
+        "batch_size": 50,
+        "use_async_envs": false
+    },
+    "wandb": {
+        "enable": true,
+        "disable_artifact": false,
+        "project": "lerobot",
+        "entity": null,
+        "notes": null,
+        "run_id": "queg8q99",
+        "mode": null,
+        "add_tags": true
+    },
+    "peft": null,
+    "use_rabc": false,
+    "rabc_progress_path": null,
+    "rabc_kappa": 0.01,
+    "rabc_epsilon": 1e-06,
+    "rabc_head_mode": "sparse",
+    "rename_map": {},
+    "checkpoint_path": null
+}