docker/scripts/backup_scheduler.py

#!/usr/bin/env python3
"""Start CommitScheduler for periodic backup to HF Dataset.

If PAPERCLIP_DATA_REPO_ID and HF_TOKEN are set, initializes CommitScheduler
to push PAPERCLIP_HOME to a private Dataset repo every 5 minutes.
Then blocks indefinitely so the scheduler background thread stays alive.

If env vars are missing, exits immediately (no scheduler).
"""
import os
import signal
import sys

from huggingface_hub import CommitScheduler


def _get_env(name, default=""):
    """Read env var at call time (not import time) so tests can monkeypatch."""
    return os.environ.get(name, default)


def _get_instance_name():
    """Derive sanitized instance name from SPACE_ID or fallback."""
    name = (
        _get_env("SPACE_ID")
        or _get_env("PAPERCLIP_INSTANCE_NAME")
        or _get_env("PAPERCLIP_INSTANCE_ID", "default")
    )
    return name.replace("/", "-")


def main():
    paperclip_home = _get_env("PAPERCLIP_HOME", "/paperclip")
    data_repo_id = _get_env("PAPERCLIP_DATA_REPO_ID")
    hf_token = _get_env("HF_TOKEN") or _get_env("HUGGING_FACE_HUB_TOKEN")

    if not data_repo_id or not hf_token:
        print("backup_scheduler: PAPERCLIP_DATA_REPO_ID or HF_TOKEN not set, exiting")
        return

    instance_name = _get_instance_name()

    scheduler = CommitScheduler(
        repo_id=data_repo_id,
        repo_type="dataset",
        folder_path=paperclip_home,
        path_in_repo=f"{instance_name}/",
        every=5,
        token=hf_token,
        ignore_patterns=[
            "**/*.log",
            "logs/**",
            "cache/**",
            ".cache/**",
            "**/*.tmp",
            "**/*.pid",
        ],
        squash_history=True,
    )
    print(
        f"backup_scheduler: started for {data_repo_id}/{instance_name}/ "
        f"(every 5min)"
    )

    # Block indefinitely so CommitScheduler background thread stays alive.
    # SIGTERM from container stop will interrupt signal.pause().
    signal.pause()


if __name__ == "__main__":
    main()