SO101_Commands.txt

FAN: sudo jetson_clocks --fan
sudo jetson_clocks
================================================================================
SO101 ROBOT - PRE-FLIGHT CHECKLIST (DO THIS FIRST!)
================================================================================

Step 1: Check Serial Port Permissions (Robot & Teleop)

Virtual environment:
source .venv/bin/activate

------------------------------------------------------
# This ensures you can actually talk to the hardware.
sudo chmod 777 /dev/ttyACM*
ls -la /dev/ttyACM*

Step 2: Check Camera Permissions
--------------------------------
# This fixes the "TimeoutError: Timed out waiting for frame" error.
sudo chmod 666 /dev/video*
ls -la /dev/video*

Step 3: Identify Ports (Verify they match your command)
-------------------------------------------------------
# Confirm which port is the FOLLOWER and which is the LEADER.
lerobot-find-port

# Confirm which video device is which camera.
lerobot-find-cameras opencv

Step 4: Verify Your Command Port Mapping
----------------------------------------
In your lerobot-teleoperate or lerobot-rollout command, ensure:
--robot.port=  matches the FOLLOWER port from Step 3. (Usually /dev/ttyACM0)
--teleop.port= matches the LEADER port from Step 3.   (Usually /dev/ttyACM1)
--robot.cameras='{ camera1: {index_or_path: /dev/videoX} }' matches your camera scan.

================================================================================
TELEOPERATION & RECORDING
================================================================================

# Standard Teleoperation
lerobot-teleoperate \
    --robot.type=so101_follower \
    --robot.port=/dev/ttyACM1 \
    --robot.cameras='{
        camera1: {type: opencv, index_or_path: /dev/video0, width: 320, height: 240, fps: 30},
        camera2: {type: opencv, index_or_path: /dev/video4, width: 320, height: 240, fps: 30}
    }' \
    --robot.id=so101_follower_7V \
    --teleop.type=so101_leader \
    --teleop.port=/dev/ttyACM0 \
    --teleop.id=so_101_leader_7V \
    --display_data=true

# Record new dataset (Manual Teleoperation)
lerobot-record \
    --robot.type=so101_follower \
    --robot.port=/dev/ttyACM1 \
    --robot.cameras='{
        camera1: {type: opencv, index_or_path: /dev/video0, width: 320, height: 240, fps: 30},
        camera2: {type: opencv, index_or_path: /dev/video4, width: 320, height: 240, fps: 30}
    }' \
    --robot.id=so101_follower_7V \
    --teleop.type=so101_leader \
    --teleop.port=/dev/ttyACM0 \
    --teleop.id=so_101_leader_7V \
    --dataset.repo_id=vraiRobotLab/k_duck\
    --dataset.single_task="pick and place duck" \
    --dataset.fps=30 \
    --dataset.num_episodes=50 \
    --dataset.episode_time_s=20 \
    --dataset.reset_time_s=5 \
    --dataset.root=data/k_duck \
    --dataset.streaming_encoding=true \
    --dataset.encoder_threads=2 \
    --display_data=true \
    --resume=false

# Bimanual Teleoperation
lerobot-teleoperate \
    --robot.type=bi_so_follower \
    --robot.left_arm_config.port=/dev/ttyACM6 \
    --robot.right_arm_config.port=/dev/ttyACM0 \
    --robot.left_arm_config.cameras='{
        camera1: {type: opencv, index_or_path: /dev/video2, width: 320, height: 240, fps: 30, backend: 200},
        camera2: {type: opencv, index_or_path: /dev/video4, width: 320, height: 240, fps: 30, backend: 200}
    }' \
    --robot.id=bi_so101_follower_7V \
    --teleop.type=bi_so_leader \
    --teleop.left_arm_config.port=/dev/ttyACM5 \
    --teleop.right_arm_config.port=/dev/ttyACM4 \
    --teleop.id=bi_so101_leader_7V \
    --display_data=true

# Record new bimanual dataset (Manual Teleoperation)
lerobot-record \
    --robot.type=bi_so_follower \
    --robot.left_arm_config.port=/dev/ttyACM6 \
    --robot.right_arm_config.port=/dev/ttyACM0 \
    --robot.left_arm_config.cameras='{
        camera1: {type: opencv, index_or_path: /dev/video2, width: 320, height: 240, fps: 30}
    }' \
    --robot.right_arm_config.cameras='{
        camera2: {type: opencv, index_or_path: /dev/video4, width: 320, height: 240, fps: 30}
    }' \
    --robot.id=bi_so101_follower_7V \
    --teleop.type=bi_so_leader \
    --teleop.left_arm_config.port=/dev/ttyACM2 \
    --teleop.right_arm_config.port=/dev/ttyACM1 \
    --teleop.id=bi_so101_leader_7V \
    --dataset.repo_id=NLTuan/bi_transfer \
    --dataset.single_task="Transfer stick from red to green strip" \
    --dataset.fps=30 \
    --dataset.num_episodes=15 \
    --dataset.episode_time_s=30 \
    --dataset.reset_time_s=5 \
    --dataset.root=data/bi_transfer \
    --dataset.streaming_encoding=true \
    --dataset.encoder_threads=2 \
    --display_data=true \
    --resume=true

================================================================================
TRAINING COMMANDS (lerobot-train)
================================================================================

# Basic Training (ACT)
lerobot-train \
    --policy.type=act \
    --dataset.repo_id=ThavT/red_block_in_tape \
    --batch_size=64 \
    --steps=20000 \
    --output_dir=outputs/train/red_block_in_tape_act \
    --job_name=act_red_block_training \
    --policy.push_to_hub=true \
    --policy.repo_id=NLTuan/act_red_block_in_tape \
    --rename_map='{"observation.images.cam_0": "observation.images.camera1", "observation.images.cam_1": "observation.images.camera2"}' \
    --wandb.enable=true \
    --wandb.project=lerobot_red_block \
    --seed=42 \
    --log_freq=50

# Fine-Tuning with Regularization (LoRA & Weight Decay)
# - Use LoRA for heavy models (SmolVLA) to save memory and avoid overfitting.
lerobot-train \
    --dataset.repo_id=NLTuan/red_blue_block_cleaned \
    --policy.type=act \
    --peft.method_type=LORA \
    --peft.r=16 \
    --optimizer.weight_decay=1e-2 \
    --optimizer.grad_clip_norm=1.0 \
    --policy.push_to_hub=false \
    --steps=100000 \
    --batch_size=8 \
    --wandb.enable=true

# Enabling Data Augmentation (Color Jitter & Crops)
# - Highly recommended once the model has learned the basic task.
lerobot-train \
    --dataset.repo_id=NLTuan/red_blue_block_cleaned \
    --policy.type=act \
    --policy.image_transforms.enable=true \
    --policy.image_transforms.max_num_transforms=3 \
    --policy.push_to_hub=false \
    --steps=100000 \
    --batch_size=8

# Auto-Evaluation during training
# - Records test episodes periodically to check Success Rate.
lerobot-train \
    --dataset.repo_id=NLTuan/red_blue_block_cleaned \
    --policy.type=act \
    --eval.n_episodes=5 \
    --eval_freq=5000 \
    --policy.push_to_hub=false \
    --steps=100000 \
    --batch_size=8

# Finetuning an existing Hub Model
# - Loads weights from a specific repo and continues training on new data.
lerobot-train \
    --policy.path=NLTuan/act-blue-box-lr4e-5 \
    --dataset.repo_id=ThavT/red_block_in_tape \
    --steps=20000 \
    --batch_size=32 \
    --optimizer.lr=1e-5 \
    --rename_map='{"observation.images.cam_0": "observation.images.camera1", "observation.images.cam_1": "observation.images.camera2"}' \
    --policy.push_to_hub=true \
    --policy.repo_id=NLTuan/act_red_block_finetuned

================================================================================
ROLLOUT MODES (lerobot-rollout)
================================================================================

--- MODE 1: BASE (EVALUATION ONLY) ---
# No data is recorded. Used just to test/watch the policy.
# NOTE: Cannot use any --dataset.* flags here.
lerobot-rollout \
    --strategy.type=base \
    --robot.type=so101_follower \
    --robot.port=/dev/ttyACM1 \
    --robot.id=so101_follower_7V \
    --robot.cameras='{
        camera1: {type: opencv, index_or_path: /dev/video2, width: 640, height: 480, fps: 30, backend: 200, fourcc: MJPG},
        camera2: {type: opencv, index_or_path: /dev/video4, width: 640, height: 480, fps: 30, backend: 200, fourcc: MJPG}
    }' \
    --policy.path=NLTuan/act_j_duck_tony_train \
    --policy.pretrained_revision=step_50000 \
    --display_data=true \
    --policy.n_action_steps=1 \
    --policy.temporal_ensemble_coeff=0.1 




--- MODE 1B: BIMANUAL BASE (ACT PICK + STACK) ---
# Bimanual rollout for ThavT/act_bi_pick_stack_obs2.
# The policy expects camera feature names camera1 and camera2.
# Use OpenCV V4L2 backend 200 for /dev/video0 and /dev/video2.
# To pin a Hugging Face model revision, add this after --policy.path:
#     --policy.pretrained_revision=step_018000 \
# Example revision values for this repo: step_002000 ... step_018000.
uv run lerobot-rollout \
    --strategy.type=base \
    --policy.path=ThavT/act_bi_pick_stack_obs2 \
    --robot.type=bi_so_follower \
    --robot.left_arm_config.port=/dev/ttyACM3 \
    --robot.right_arm_config.port=/dev/ttyACM0 \
    --robot.id=bi_so101_follower_7V \
    --robot.cameras='{ camera1: {type: opencv, index_or_path: /dev/video0, width: 320, height: 240, fps: 30, backend: 200}, camera2: {type: opencv, index_or_path: /dev/video2, width: 320, height: 240, fps: 30, backend: 200} }' \
    --task="pick stack" \
    --duration=60 \
    --display_data=true

# Same rollout pinned to a specific Hugging Face model revision.
# This repo stores checkpoints on step branches; step_018000 contains the processor files.
uv run lerobot-rollout \
    --strategy.type=base \
    --policy.path=ThavT/act_bi_pick_stack_obs2 \
    --policy.pretrained_revision=step_002000 \
    --robot.type=bi_so_follower \
    --robot.left_arm_config.port=/dev/ttyACM3 \
    --robot.right_arm_config.port=/dev/ttyACM0 \
    --robot.id=bi_so101_follower_7V \
    --robot.cameras='{ camera1: {type: opencv, index_or_path: /dev/video0, width: 320, height: 240, fps: 30, backend: 200}, camera2: {type: opencv, index_or_path: /dev/video2, width: 320, height: 240, fps: 30, backend: 200} }' \
    --task="pick stack" \
    --duration=60 \
    --display_data=true

--- MODE 2: SENTRY (RECORD EVALUATION) ---
# Policy drives the robot and ALWAYS records the data to a dataset.
lerobot-rollout \
    --strategy.type=sentry \
    --dataset.repo_id=NLTuan/eval_results \
    --dataset.num_episodes=10 \
    --dataset.episode_time_s=600 \
    --robot.type=so101_follower \
    --robot.port=/dev/ttyACM0 \
    --robot.id=so101_follower_7V \
    --robot.cameras='{
        camera1: {type: opencv, index_or_path: /dev/video3, width: 640, height: 480, fps: 30},
        camera2: {type: opencv, index_or_path: /dev/video6, width: 640, height: 480, fps: 30}
    }' \
    --policy.path=NLTuan/act_red_blue_5e4 \
    --display_data=true

--- MODE 3: HIGHLIGHT (TRIGGERED RECORDING) ---
# Policy drives. Press 'S' to save the last 10 seconds.
lerobot-rollout \
    --strategy.type=highlight \
    --strategy.ring_buffer_seconds=10.0 \
    --dataset.repo_id=NLTuan/policy_highlights \
    --robot.type=so101_follower \
    --robot.port=/dev/ttyACM0 \
    --robot.id=so101_follower_7V \
    --robot.cameras='{
        camera1: {type: opencv, index_or_path: /dev/video3, width: 640, height: 480, fps: 30},
        camera2: {type: opencv, index_or_path: /dev/video6, width: 640, height: 480, fps: 30}
    }' \
    --policy.path=NLTuan/act_red_blue_5e4 \
    --display_data=true

--- MODE 4: DAGGER (HUMAN-IN-THE-LOOP) ---
# Interactive: Policy drives, but you can intervene with the leader arm.
lerobot-rollout \
    --strategy.type=dagger \
    --dataset.repo_id=NLTuan/dagger_data \
    --robot.type=so101_follower \
    --robot.port=/dev/ttyACM0 \
    --robot.id=so101_follower_7V \
    --teleop.type=so101_leader \
    --teleop.port=/dev/ttyACM1 \
    --robot.cameras='{
        camera1: {type: opencv, index_or_path: /dev/video3, width: 640, height: 480, fps: 30},
        camera2: {type: opencv, index_or_path: /dev/video6, width: 640, height: 480, fps: 30}
    }' \
    --policy.path=NLTuan/act_red_blue_5e4 \
    --display_data=true

================================================================================
SPECIFIC POLICY ARCHITECTURES (FOR CHUNKING MODELS)
================================================================================

# 1. SMOLVLA (VLA Model)
# - Requires: 'transformers' (pip install 'lerobot[smolvla]')
lerobot-rollout \
    --strategy.type=base \
    --policy.path=NLTuan/smolvla_clean_lora \
    --dataset.episode_time_s=600 \
    --policy.device=cuda \
    --display_data=true

# 2. DIFFUSION POLICY
# - Key Flags: --policy.num_inference_steps=5 (lower = faster)
#              --policy.noise_scheduler_type=DDIM
lerobot-rollout \
    --strategy.type=base \
    --policy.path=NLTuan/diffusion_model \
    --policy.num_inference_steps=5 \
    --policy.noise_scheduler_type=DDIM \
    --policy.device=cuda \
    --display_data=true

# 3. OVERRIDING CHUNK SIZE / ACTION EXECUTION
# - Flag: --policy.n_action_steps=20 (set how many actions to execute at once)
lerobot-rollout \
    --strategy.type=base \
    --policy.path=NLTuan/act_red_blue_5e4 \
    --policy.n_action_steps=20 \
    --display_data=true

================================================================================
ASYNC INFERENCE COMMANDS (High Performance)
================================================================================

# 1. Start Policy Server (Compute)
python -m lerobot.async_inference.policy_server --host=127.0.0.1 --port=8080

# 2. Start Robot Client (Control)
python -m lerobot.async_inference.robot_client \
    --server_address=127.0.0.1:8080 \
    --robot.type=so101_follower \
    --robot.port=/dev/ttyACM0 \
    --robot.cameras='{
        camera1: {type: opencv, index_or_path: /dev/video3, width: 640, height: 480, fps: 30},
        camera2: {type: opencv, index_or_path: /dev/video6, width: 640, height: 480, fps: 30}
    }' \
    --policy_type=act \
    --pretrained_name_or_path=NLTuan/act_block_in_tape \
    --actions_per_chunk=50

================================================================================
UTILITIES & TROUBLESHOOTING
================================================================================

# Replay recorded data
lerobot-replay --dataset.repo_id=NLTuan/dataset --dataset.episode_index=0

# Visualize dataset
lerobot-dataset-viz --dataset.repo_id=NLTuan/dataset

# Fix Camera/Serial permissions
sudo chmod 666 /dev/video*
sudo chmod 666 /dev/ttyACM*

# Calibration
lerobot-calibrate --robot.type=so101_follower --robot.port=/dev/ttyACM0

# Bimanual Calibration (Follower Arms)
lerobot-calibrate \
    --robot.type=bi_so_follower \
    --robot.left_arm_config.port=/dev/ttyACM6 \
    --robot.right_arm_config.port=/dev/ttyACM0 \
    --robot.id=bi_so101_follower_7V


# Bimanual Calibration (Leader Arms)
lerobot-calibrate \
    --teleop.type=bi_so_leader \
    --teleop.left_arm_config.port=/dev/ttyACM5 \
    --teleop.right_arm_config.port=/dev/ttyACM4 \
    --teleop.id=bi_so101_leader_7V