Spaces:
Sleeping
Sleeping
File size: 3,497 Bytes
59653ee | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 | """Upload ElRobot parquet dataset to HuggingFace Hub.
Usage:
uv run python scripts/upload_dataset_hf.py --repo-id YOUR_USERNAME/elrobot-pickplace
Requires: huggingface-cli login (write token)
"""
from __future__ import annotations
import argparse
from pathlib import Path
from huggingface_hub import HfApi, create_repo
def parse_args():
p = argparse.ArgumentParser()
p.add_argument("--repo-id", required=True, help="HF dataset repo (e.g. venaychawda/elrobot-pickplace)")
p.add_argument("--parquets-dir", type=Path, default=Path("/home/venay/datasets/normacore"),
help="Directory containing parquet files")
p.add_argument("--private", action="store_true", help="Make the dataset private")
return p.parse_args()
def main():
args = parse_args()
api = HfApi()
parquets = sorted(args.parquets_dir.glob("*.parquet"))
if not parquets:
raise SystemExit(f"No parquet files in {args.parquets_dir}")
print(f"Found {len(parquets)} parquet files:")
total_bytes = 0
for p in parquets:
size = p.stat().st_size
total_bytes += size
print(f" {p.name} ({size/1e6:.1f} MB)")
print(f" Total: {total_bytes/1e6:.1f} MB")
print(f"\nCreating dataset repo: {args.repo_id}")
create_repo(args.repo_id, repo_type="dataset", private=args.private, exist_ok=True)
print("Uploading parquet files...")
api.upload_folder(
repo_id=args.repo_id,
folder_path=str(args.parquets_dir),
path_in_repo="data",
repo_type="dataset",
allow_patterns="*.parquet",
)
readme = f"""---
tags:
- robotics
- elrobot
- normacore
- smolvla
- pick-and-place
license: apache-2.0
---
# ElRobot Pick & Place Dataset
Teleoperation demonstrations recorded on NormaCore ElRobot (8-DOF + gripper).
## Robot
- **NormaCore ElRobot** — 7 rotational joints + 1 gripper (8 ST3215 servos)
- **Controller:** Raspberry Pi 5
- **Cameras:** 2x USB RGB (224x224)
## Dataset Format (NormaCore custom parquet)
One row = one frame. Schema:
| Column | Type | Description |
|---|---|---|
| episode_start_ns | uint64 | Episode identifier |
| timestamp_ns_since_episode_start | uint64 | Per-episode time |
| joints | list<struct> | 8 joints: position_norm, goal_norm, range_min, range_max, current_ma, velocity |
| images | list<struct{{jpeg:binary}}> | 2 embedded JPEG images per frame |
| task | string | Natural language task instruction |
## Tasks Recorded
- push the block forward (4 episodes)
- pick up the block (4 episodes)
- pick up the pen (2 episodes)
- pick up the block and put it down (1 episode)
- pick up the bottle cap (1 episode)
## Stats
- **Episodes:** 12
- **Frames:** 5,099
- **Joints:** 8 (normalized 0-1)
- **Images:** 2 per frame (224x224 JPEG)
## Training
```bash
uv run python scripts/train_elrobot.py \\
--parquets data/*.parquet \\
--base-checkpoint LBST/t01_pick_and_place \\
--steps 5000 --batch-size 32
```
"""
readme_path = args.parquets_dir.parent / "dataset_readme.md"
readme_path.write_text(readme)
api.upload_file(
path_or_fileobj=str(readme_path),
path_in_repo="README.md",
repo_id=args.repo_id,
repo_type="dataset",
)
readme_path.unlink()
print(f"\nDataset uploaded: https://huggingface.co/datasets/{args.repo_id}")
if __name__ == "__main__":
main()
|