scripts/upload_to_hf.py

#!/usr/bin/env python3
"""Upload this archive to Hugging Face Hub.

Requires an HF token with write access to the target namespace.
Default target: SlayerLab/slayer-gpt-tokenizer-model
"""

from __future__ import annotations

import argparse
from pathlib import Path

from huggingface_hub import HfApi, create_repo


def main() -> None:
    parser = argparse.ArgumentParser()
    parser.add_argument("--repo-id", default="SlayerLab/slayer-gpt-tokenizer-model")
    parser.add_argument("--private", action="store_true")
    parser.add_argument("--repo-type", default="model", choices=["model", "dataset", "space"])
    args = parser.parse_args()

    root = Path(__file__).resolve().parents[1]
    create_repo(
        args.repo_id,
        repo_type=args.repo_type,
        private=args.private,
        exist_ok=True,
    )

    commit = HfApi().upload_folder(
        repo_id=args.repo_id,
        repo_type=args.repo_type,
        folder_path=str(root),
        commit_message="Upload Slayer GPT tokenizer model archive",
        ignore_patterns=[
            ".git/*",
            ".venv/*",
            "__pycache__/*",
            "**/__pycache__/*",
            ".DS_Store",
        ],
    )
    print(commit)


if __name__ == "__main__":
    main()