#!/usr/bin/env python3 """Upload the dataset folder to the Hugging Face Hub. Usage: python upload_to_hf.py --repo-id YOURNAME/pendulum-cjepa-3k-clevrer \ --root /home/user/pendulum3k/dataset Authenticate first: either `huggingface-cli login` or export HF_TOKEN=... Files: - everything under --root is uploaded except .tmp files and __pycache__. """ import argparse from huggingface_hub import HfApi def main(): ap = argparse.ArgumentParser() ap.add_argument("--repo-id", required=True, help="e.g. jimchen2/pendulum-cjepa-3k-clevrer") ap.add_argument("--root", default="/home/user/pendulum3k/dataset") ap.add_argument("--private", action="store_true") ap.add_argument("--readme", default="/home/user/pendulum3k/README.md") args = ap.parse_args() api = HfApi() api.create_repo(args.repo_id, repo_type="dataset", private=args.private, exist_ok=True) api.upload_folder(repo_id=args.repo_id, repo_type="dataset", folder_path=args.root, path_in_repo="", ignore_patterns=["*.tmp*", "__pycache__", "*.pyc"]) if args.readme: api.upload_file(repo_id=args.repo_id, repo_type="dataset", path_or_fileobj=args.readme, path_in_repo="README.md") print(f"uploaded to https://huggingface.co/datasets/{args.repo_id}") if __name__ == "__main__": main()