Nottybro's picture
Error reports service (/cos2/)
9df8967 verified
Raw History Blame Contribute Delete
4.12 kB
"""
Runs the Space: nginx in front (port 7860), the Kimi proxy (uvicorn on 8000) and the Captain of Sea
server (27777), restarting either if it stops. The Space's disk is wiped on every restart, so the
captains' saves are restored from a private dataset at start and backed up to it every few minutes
(needs the HF_TOKEN secret; SAVES_REPO names the dataset), along with the game's error reports
(reports.py, reports/errors.json).
"""
import os
import signal
import subprocess
import sys
import threading
import time
SAVES = os.environ.get("CAPTAIN_SAVES", "/home/user/saves")
REPO = os.environ.get("SAVES_REPO", "Nottybro/captain-of-sea-saves")
TOKEN = os.environ.get("HF_TOKEN")
BACKUP_EVERY = int(os.environ.get("BACKUP_SECONDS", "300"))
GAME = ["./server/CaptainServer.x86_64", "-batchmode", "-nographics", "-logFile", "-"]
PROXY = [sys.executable, "-m", "uvicorn", "app:app", "--host", "127.0.0.1", "--port", "8000"]
REPORTS = [sys.executable, "-u", "reports.py"]
REPORT_FILE = "reports/errors.json"
stopping = threading.Event()
uploaded = {} # file name -> mtime last backed up
def log(msg):
print("[run] " + msg, flush=True)
def restore():
os.makedirs(SAVES, exist_ok=True)
if not TOKEN:
log("no HF_TOKEN secret: saves won't outlive a restart")
return
from huggingface_hub import HfApi, snapshot_download
try:
HfApi(token=TOKEN).create_repo(REPO, repo_type="dataset", private=True, exist_ok=True)
snapshot_download(REPO, repo_type="dataset", local_dir=SAVES, token=TOKEN, allow_patterns=["*.json", REPORT_FILE])
except Exception as e:
log("restore failed: %s" % e)
for f in os.listdir(SAVES):
if f.endswith(".json"):
uploaded[f] = os.path.getmtime(os.path.join(SAVES, f))
log("restored %d captains" % len(uploaded))
def backup():
if not TOKEN:
return
from huggingface_hub import CommitOperationAdd, HfApi
changed = []
for f in os.listdir(SAVES) + [REPORT_FILE]:
p = os.path.join(SAVES, f)
if f.endswith(".json") and os.path.isfile(p) and uploaded.get(f) != os.path.getmtime(p):
changed.append((f, os.path.getmtime(p)))
if not changed:
return
try:
ops = [CommitOperationAdd(path_in_repo=f, path_or_fileobj=os.path.join(SAVES, f)) for f, _ in changed]
HfApi(token=TOKEN).create_commit(REPO, repo_type="dataset", operations=ops, commit_message="%d captains" % len(ops))
for f, m in changed:
uploaded[f] = m
log("backed up %d captains" % len(changed))
except Exception as e:
log("backup failed: %s" % e)
def backups():
while not stopping.wait(BACKUP_EVERY):
backup()
def keep_running(name, cmd, cwd="/app"):
"""Start cmd and start it again whenever it stops (until the Space shuts down)."""
def loop():
while not stopping.is_set():
log("starting " + name)
p = subprocess.Popen(cmd, cwd=cwd)
procs[name] = p
p.wait()
if stopping.is_set():
return
log("%s stopped (code %s), restarting in 3 s" % (name, p.returncode))
time.sleep(3)
threading.Thread(target=loop, daemon=True).start()
procs = {}
def shutdown(*_):
log("shutting down")
stopping.set()
for p in procs.values():
if p.poll() is None:
p.send_signal(signal.SIGTERM)
time.sleep(4) # the game saves everyone on the way down
backup()
sys.exit(0)
if __name__ == "__main__":
signal.signal(signal.SIGTERM, shutdown)
signal.signal(signal.SIGINT, shutdown)
restore()
keep_running("proxy", PROXY)
keep_running("game", GAME)
keep_running("reports", REPORTS)
threading.Thread(target=backups, daemon=True).start()
nginx = subprocess.Popen(["nginx", "-c", "/app/nginx.conf", "-g", "daemon off;"])
procs["nginx"] = nginx
nginx.wait()
log("nginx stopped (code %s)" % nginx.returncode)
shutdown()