#!/usr/bin/env python3 """Start CommitScheduler for periodic backup to HF Dataset. If PAPERCLIP_DATA_REPO_ID and HF_TOKEN are set, initializes CommitScheduler to push PAPERCLIP_HOME to a private Dataset repo every 5 minutes. Then blocks indefinitely so the scheduler background thread stays alive. If env vars are missing, exits immediately (no scheduler). """ import os import signal import sys from huggingface_hub import CommitScheduler def _get_env(name, default=""): """Read env var at call time (not import time) so tests can monkeypatch.""" return os.environ.get(name, default) def _get_instance_name(): """Derive sanitized instance name from SPACE_ID or fallback.""" name = ( _get_env("SPACE_ID") or _get_env("PAPERCLIP_INSTANCE_NAME") or _get_env("PAPERCLIP_INSTANCE_ID", "default") ) return name.replace("/", "-") def main(): paperclip_home = _get_env("PAPERCLIP_HOME", "/paperclip") data_repo_id = _get_env("PAPERCLIP_DATA_REPO_ID") hf_token = _get_env("HF_TOKEN") or _get_env("HUGGING_FACE_HUB_TOKEN") if not data_repo_id or not hf_token: print("backup_scheduler: PAPERCLIP_DATA_REPO_ID or HF_TOKEN not set, exiting") return instance_name = _get_instance_name() scheduler = CommitScheduler( repo_id=data_repo_id, repo_type="dataset", folder_path=paperclip_home, path_in_repo=f"{instance_name}/", every=5, token=hf_token, ignore_patterns=[ "**/*.log", "logs/**", "cache/**", ".cache/**", "**/*.tmp", "**/*.pid", ], squash_history=True, ) print( f"backup_scheduler: started for {data_repo_id}/{instance_name}/ " f"(every 5min)" ) # Block indefinitely so CommitScheduler background thread stays alive. # SIGTERM from container stop will interrupt signal.pause(). signal.pause() if __name__ == "__main__": main()