File size: 1,418 Bytes
9b2853a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 | #!/usr/bin/env python3
"""Upload the dataset folder to the Hugging Face Hub.
Usage:
python upload_to_hf.py --repo-id YOURNAME/pendulum-cjepa-3k-clevrer \
--root /home/user/pendulum3k/dataset
Authenticate first: either `huggingface-cli login` or export HF_TOKEN=...
Files:
- everything under --root is uploaded except .tmp files and __pycache__.
"""
import argparse
from huggingface_hub import HfApi
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--repo-id", required=True,
help="e.g. jimchen2/pendulum-cjepa-3k-clevrer")
ap.add_argument("--root", default="/home/user/pendulum3k/dataset")
ap.add_argument("--private", action="store_true")
ap.add_argument("--readme", default="/home/user/pendulum3k/README.md")
args = ap.parse_args()
api = HfApi()
api.create_repo(args.repo_id, repo_type="dataset", private=args.private,
exist_ok=True)
api.upload_folder(repo_id=args.repo_id, repo_type="dataset",
folder_path=args.root, path_in_repo="",
ignore_patterns=["*.tmp*", "__pycache__", "*.pyc"])
if args.readme:
api.upload_file(repo_id=args.repo_id, repo_type="dataset",
path_or_fileobj=args.readme, path_in_repo="README.md")
print(f"uploaded to https://huggingface.co/datasets/{args.repo_id}")
if __name__ == "__main__":
main()
|