File size: 4,117 Bytes
9df8967
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
"""

Runs the Space: nginx in front (port 7860), the Kimi proxy (uvicorn on 8000) and the Captain of Sea

server (27777), restarting either if it stops. The Space's disk is wiped on every restart, so the

captains' saves are restored from a private dataset at start and backed up to it every few minutes

(needs the HF_TOKEN secret; SAVES_REPO names the dataset), along with the game's error reports

(reports.py, reports/errors.json).

"""
import os
import signal
import subprocess
import sys
import threading
import time

SAVES = os.environ.get("CAPTAIN_SAVES", "/home/user/saves")
REPO = os.environ.get("SAVES_REPO", "Nottybro/captain-of-sea-saves")
TOKEN = os.environ.get("HF_TOKEN")
BACKUP_EVERY = int(os.environ.get("BACKUP_SECONDS", "300"))

GAME = ["./server/CaptainServer.x86_64", "-batchmode", "-nographics", "-logFile", "-"]
PROXY = [sys.executable, "-m", "uvicorn", "app:app", "--host", "127.0.0.1", "--port", "8000"]
REPORTS = [sys.executable, "-u", "reports.py"]
REPORT_FILE = "reports/errors.json"

stopping = threading.Event()
uploaded = {}  # file name -> mtime last backed up


def log(msg):
    print("[run] " + msg, flush=True)


def restore():
    os.makedirs(SAVES, exist_ok=True)
    if not TOKEN:
        log("no HF_TOKEN secret: saves won't outlive a restart")
        return
    from huggingface_hub import HfApi, snapshot_download
    try:
        HfApi(token=TOKEN).create_repo(REPO, repo_type="dataset", private=True, exist_ok=True)
        snapshot_download(REPO, repo_type="dataset", local_dir=SAVES, token=TOKEN, allow_patterns=["*.json", REPORT_FILE])
    except Exception as e:
        log("restore failed: %s" % e)
    for f in os.listdir(SAVES):
        if f.endswith(".json"):
            uploaded[f] = os.path.getmtime(os.path.join(SAVES, f))
    log("restored %d captains" % len(uploaded))


def backup():
    if not TOKEN:
        return
    from huggingface_hub import CommitOperationAdd, HfApi
    changed = []
    for f in os.listdir(SAVES) + [REPORT_FILE]:
        p = os.path.join(SAVES, f)
        if f.endswith(".json") and os.path.isfile(p) and uploaded.get(f) != os.path.getmtime(p):
            changed.append((f, os.path.getmtime(p)))
    if not changed:
        return
    try:
        ops = [CommitOperationAdd(path_in_repo=f, path_or_fileobj=os.path.join(SAVES, f)) for f, _ in changed]
        HfApi(token=TOKEN).create_commit(REPO, repo_type="dataset", operations=ops, commit_message="%d captains" % len(ops))
        for f, m in changed:
            uploaded[f] = m
        log("backed up %d captains" % len(changed))
    except Exception as e:
        log("backup failed: %s" % e)


def backups():
    while not stopping.wait(BACKUP_EVERY):
        backup()


def keep_running(name, cmd, cwd="/app"):
    """Start cmd and start it again whenever it stops (until the Space shuts down)."""
    def loop():
        while not stopping.is_set():
            log("starting " + name)
            p = subprocess.Popen(cmd, cwd=cwd)
            procs[name] = p
            p.wait()
            if stopping.is_set():
                return
            log("%s stopped (code %s), restarting in 3 s" % (name, p.returncode))
            time.sleep(3)
    threading.Thread(target=loop, daemon=True).start()


procs = {}


def shutdown(*_):
    log("shutting down")
    stopping.set()
    for p in procs.values():
        if p.poll() is None:
            p.send_signal(signal.SIGTERM)
    time.sleep(4)  # the game saves everyone on the way down
    backup()
    sys.exit(0)


if __name__ == "__main__":
    signal.signal(signal.SIGTERM, shutdown)
    signal.signal(signal.SIGINT, shutdown)
    restore()
    keep_running("proxy", PROXY)
    keep_running("game", GAME)
    keep_running("reports", REPORTS)
    threading.Thread(target=backups, daemon=True).start()
    nginx = subprocess.Popen(["nginx", "-c", "/app/nginx.conf", "-g", "daemon off;"])
    procs["nginx"] = nginx
    nginx.wait()
    log("nginx stopped (code %s)" % nginx.returncode)
    shutdown()