scripts/upload_data_to_hf.py

"""Upload the feature store parquet to the companion HF Dataset repo.

HF Spaces silently drop files larger than 10 MB on direct upload. The
recommended pattern is therefore to host large data files in a Dataset
repo and have the Space download them at runtime via `hf_hub_download`.

PREREQUISITE — create the Dataset once on https://huggingface.co/new-dataset
    Owner: KLEB38
    Name: oc-p8-features
    Visibility: public (or private + HF_TOKEN secret on the Space)

Usage (PowerShell):
    $env:HF_TOKEN = "hf_xxx..."
    uv run python scripts/upload_data_to_hf.py
"""

from __future__ import annotations

import os
from pathlib import Path

from huggingface_hub import CommitOperationAdd, HfApi

REPO_ID = "KLEB38/oc-p8-features"
REPO_TYPE = "dataset"
PARQUET_PATH = Path("data/features_store.parquet")


def main() -> None:
    token = os.environ.get("HF_TOKEN")
    if not token:
        raise SystemExit("HF_TOKEN env var is required")

    if not PARQUET_PATH.exists():
        raise SystemExit(
            f"{PARQUET_PATH} not found - run "
            "`uv run python scripts/build_feature_store.py` first."
        )

    size_mb = PARQUET_PATH.stat().st_size / (1024 * 1024)
    print(f"Local file: {PARQUET_PATH} ({size_mb:.1f} MB)")

    api = HfApi(token=token)
    print(f"Authenticated as: {api.whoami()['name']}")

    print(f"\nUploading to {REPO_TYPE} repo: {REPO_ID}")
    commit_info = api.create_commit(
        repo_id=REPO_ID,
        repo_type=REPO_TYPE,
        operations=[
            CommitOperationAdd(
                path_in_repo="features_store.parquet",
                path_or_fileobj=str(PARQUET_PATH),
            ),
        ],
        commit_message="Update features store parquet",
    )
    print(f"Commit OID: {commit_info.oid}")
    print(f"Commit URL: {commit_info.commit_url}")

    print("\nFiles AFTER upload:")
    for f in api.list_repo_files(repo_id=REPO_ID, repo_type=REPO_TYPE):
        marker = "  <-- TARGET" if f == "features_store.parquet" else ""
        print(f"  - {f}{marker}")


if __name__ == "__main__":
    main()