| |
| """Upload the dataset folder to the Hugging Face Hub. |
| |
| Usage: |
| python upload_to_hf.py --repo-id YOURNAME/pendulum-cjepa-3k-clevrer \ |
| --root /home/user/pendulum3k/dataset |
| |
| Authenticate first: either `huggingface-cli login` or export HF_TOKEN=... |
| Files: |
| - everything under --root is uploaded except .tmp files and __pycache__. |
| """ |
| import argparse |
| from huggingface_hub import HfApi |
|
|
|
|
| def main(): |
| ap = argparse.ArgumentParser() |
| ap.add_argument("--repo-id", required=True, |
| help="e.g. jimchen2/pendulum-cjepa-3k-clevrer") |
| ap.add_argument("--root", default="/home/user/pendulum3k/dataset") |
| ap.add_argument("--private", action="store_true") |
| ap.add_argument("--readme", default="/home/user/pendulum3k/README.md") |
| args = ap.parse_args() |
|
|
| api = HfApi() |
| api.create_repo(args.repo_id, repo_type="dataset", private=args.private, |
| exist_ok=True) |
| api.upload_folder(repo_id=args.repo_id, repo_type="dataset", |
| folder_path=args.root, path_in_repo="", |
| ignore_patterns=["*.tmp*", "__pycache__", "*.pyc"]) |
| if args.readme: |
| api.upload_file(repo_id=args.repo_id, repo_type="dataset", |
| path_or_fileobj=args.readme, path_in_repo="README.md") |
| print(f"uploaded to https://huggingface.co/datasets/{args.repo_id}") |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
