"""Upload ElRobot parquet dataset to HuggingFace Hub. Usage: uv run python scripts/upload_dataset_hf.py --repo-id YOUR_USERNAME/elrobot-pickplace Requires: huggingface-cli login (write token) """ from __future__ import annotations import argparse from pathlib import Path from huggingface_hub import HfApi, create_repo def parse_args(): p = argparse.ArgumentParser() p.add_argument("--repo-id", required=True, help="HF dataset repo (e.g. venaychawda/elrobot-pickplace)") p.add_argument("--parquets-dir", type=Path, default=Path("/home/venay/datasets/normacore"), help="Directory containing parquet files") p.add_argument("--private", action="store_true", help="Make the dataset private") return p.parse_args() def main(): args = parse_args() api = HfApi() parquets = sorted(args.parquets_dir.glob("*.parquet")) if not parquets: raise SystemExit(f"No parquet files in {args.parquets_dir}") print(f"Found {len(parquets)} parquet files:") total_bytes = 0 for p in parquets: size = p.stat().st_size total_bytes += size print(f" {p.name} ({size/1e6:.1f} MB)") print(f" Total: {total_bytes/1e6:.1f} MB") print(f"\nCreating dataset repo: {args.repo_id}") create_repo(args.repo_id, repo_type="dataset", private=args.private, exist_ok=True) print("Uploading parquet files...") api.upload_folder( repo_id=args.repo_id, folder_path=str(args.parquets_dir), path_in_repo="data", repo_type="dataset", allow_patterns="*.parquet", ) readme = f"""--- tags: - robotics - elrobot - normacore - smolvla - pick-and-place license: apache-2.0 --- # ElRobot Pick & Place Dataset Teleoperation demonstrations recorded on NormaCore ElRobot (8-DOF + gripper). ## Robot - **NormaCore ElRobot** — 7 rotational joints + 1 gripper (8 ST3215 servos) - **Controller:** Raspberry Pi 5 - **Cameras:** 2x USB RGB (224x224) ## Dataset Format (NormaCore custom parquet) One row = one frame. Schema: | Column | Type | Description | |---|---|---| | episode_start_ns | uint64 | Episode identifier | | timestamp_ns_since_episode_start | uint64 | Per-episode time | | joints | list | 8 joints: position_norm, goal_norm, range_min, range_max, current_ma, velocity | | images | list | 2 embedded JPEG images per frame | | task | string | Natural language task instruction | ## Tasks Recorded - push the block forward (4 episodes) - pick up the block (4 episodes) - pick up the pen (2 episodes) - pick up the block and put it down (1 episode) - pick up the bottle cap (1 episode) ## Stats - **Episodes:** 12 - **Frames:** 5,099 - **Joints:** 8 (normalized 0-1) - **Images:** 2 per frame (224x224 JPEG) ## Training ```bash uv run python scripts/train_elrobot.py \\ --parquets data/*.parquet \\ --base-checkpoint LBST/t01_pick_and_place \\ --steps 5000 --batch-size 32 ``` """ readme_path = args.parquets_dir.parent / "dataset_readme.md" readme_path.write_text(readme) api.upload_file( path_or_fileobj=str(readme_path), path_in_repo="README.md", repo_id=args.repo_id, repo_type="dataset", ) readme_path.unlink() print(f"\nDataset uploaded: https://huggingface.co/datasets/{args.repo_id}") if __name__ == "__main__": main()