Spaces:
Paused
Paused
File size: 2,012 Bytes
8523e75 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 | #!/usr/bin/env python3
"""Start CommitScheduler for periodic backup to HF Dataset.
If PAPERCLIP_DATA_REPO_ID and HF_TOKEN are set, initializes CommitScheduler
to push PAPERCLIP_HOME to a private Dataset repo every 5 minutes.
Then blocks indefinitely so the scheduler background thread stays alive.
If env vars are missing, exits immediately (no scheduler).
"""
import os
import signal
import sys
from huggingface_hub import CommitScheduler
def _get_env(name, default=""):
"""Read env var at call time (not import time) so tests can monkeypatch."""
return os.environ.get(name, default)
def _get_instance_name():
"""Derive sanitized instance name from SPACE_ID or fallback."""
name = (
_get_env("SPACE_ID")
or _get_env("PAPERCLIP_INSTANCE_NAME")
or _get_env("PAPERCLIP_INSTANCE_ID", "default")
)
return name.replace("/", "-")
def main():
paperclip_home = _get_env("PAPERCLIP_HOME", "/paperclip")
data_repo_id = _get_env("PAPERCLIP_DATA_REPO_ID")
hf_token = _get_env("HF_TOKEN") or _get_env("HUGGING_FACE_HUB_TOKEN")
if not data_repo_id or not hf_token:
print("backup_scheduler: PAPERCLIP_DATA_REPO_ID or HF_TOKEN not set, exiting")
return
instance_name = _get_instance_name()
scheduler = CommitScheduler(
repo_id=data_repo_id,
repo_type="dataset",
folder_path=paperclip_home,
path_in_repo=f"{instance_name}/",
every=5,
token=hf_token,
ignore_patterns=[
"**/*.log",
"logs/**",
"cache/**",
".cache/**",
"**/*.tmp",
"**/*.pid",
],
squash_history=True,
)
print(
f"backup_scheduler: started for {data_repo_id}/{instance_name}/ "
f"(every 5min)"
)
# Block indefinitely so CommitScheduler background thread stays alive.
# SIGTERM from container stop will interrupt signal.pause().
signal.pause()
if __name__ == "__main__":
main()
|