File size: 8,677 Bytes
031a83d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
"""
Captain of Sea error reports. The game posts what broke in a player's browser (Assets/Game/Scripts/
ErrorReporter.cs: no names, ids or emails), and this keeps a tally per distinct error, plus how many
sessions started and the frame rates players got, in <CAPTAIN_SAVES>/reports/errors.json (backed up
with the saves by run.py). nginx sends /cos2/ here.

Crashes keep their last few reports in full (when in the session, the device's memory, the heap), so a
crash can be placed: at load, or minutes into a voyage. Sessions are counted per build, and the sessions
a fatal error ended per build too, so each build's crash rate reads straight off. "alive" pings (minutes
into a voyage, with the heap) show how long players stay and whether the heap creeps. "sail" (got aboard)
sits between a session's "hello" and its first "alive": the drop-off at each step. "goal" counts missions
done by id, so how far down the Captain's Log players get reads straight off too.

  POST /cos2/report            one report (JSON, sent as text/plain so browsers skip the CORS preflight)
  GET  /cos2/errors?key=...    the tally (key = the REPORT_KEY secret; off without it)
"""
import json
import os
import re
import signal
import sys
import threading
import time
from collections import deque
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from urllib.parse import parse_qs, urlparse

PATH = os.path.join(os.environ.get("CAPTAIN_SAVES", "/home/user/saves"), "reports", "errors.json")
KEY = os.environ.get("REPORT_KEY", "")
PORT = int(os.environ.get("REPORTS_PORT", "8001"))
MAX_BODY = 8192
MAX_ERRORS = 300        # distinct errors kept
MAX_KEYS = 20           # per breakdown (gpu, browser...); the rest count as "other"
MAX_SAMPLES = 40        # full reports kept per error
MAX_CRASHED = 3000      # crashed session ids kept per day
FATAL = re.compile(r"OOM|Could not allocate|out of bounds|unreachable|context lost|Aborted", re.I)
PER_IP = 60             # reports per address per 10 minutes

lock = threading.Lock()
dirty = False
recent = {}             # address -> deque of times (memory only, never saved)


def today():
    return time.strftime("%Y-%m-%d", time.gmtime())


def load():
    try:
        with open(PATH, encoding="utf-8") as f:
            return json.load(f)
    except Exception:
        return {"since": today(), "sessions": {}, "fps": [], "errors": {}}


def num(r, k):
    v = r.get(k)
    return round(v) if isinstance(v, (int, float)) and not isinstance(v, bool) else None


data = load()


def save():
    global dirty
    with lock:
        if not dirty:
            return
        text = json.dumps(data, separators=(",", ":"))
        dirty = False
    os.makedirs(os.path.dirname(PATH), exist_ok=True)
    tmp = PATH + ".tmp"
    with open(tmp, "w", encoding="utf-8") as f:
        f.write(text)
    os.replace(tmp, PATH)


def bump(d, key, n=1, most=MAX_KEYS):
    key = str(key or "?")[:80]
    if key not in d and len(d) >= most:
        key = "other"
    d[key] = d.get(key, 0) + n


def memband(r):
    """The device's memory, in bands (the browser rounds it anyway)."""
    m = num(r, "mem")
    if not m:
        return "?"
    for top in (1024, 2048, 3072, 4096, 6144, 8192, 16384):
        if m <= top:
            return "<=%dG" % (top // 1024)
    return ">16G"


def s(r, k, n=120):
    v = r.get(k)
    return str(v)[:n] if v is not None else ""


def record(r):
    global dirty
    kind = s(r, "kind", 12)
    with lock:
        if kind == "hello":
            day = data["sessions"].setdefault(today(), {"n": 0, "mobile": 0, "q": {}, "gpu": {}, "dev": {}, "os": {}})
            day["n"] += 1
            day["mobile"] += 1 if r.get("mob") else 0
            for k in ("q", "gpu", "dev", "os", "v", "mem"):
                bump(day.setdefault(k, {}), memband(r) if k == "mem" else s(r, k))
        elif kind == "alive":
            # A ping at minute 1, 3, 5, 10, 20, 30... of a voyage: how long players stay, and the heap then.
            day = data.setdefault("alive", {}).setdefault(today(), {})
            bump(day.setdefault(s(r, "v"), {}), num(r, "min"))
            data.setdefault("heap", []).append({k: r.get(k) for k in ("min", "heap", "q", "mob", "mode", "mem", "v")}
                                               | {"gpu": s(r, "gpu", 60), "day": today()})
            del data["heap"][:-800]
        elif kind == "sail":
            # Got aboard a ship: between "hello" (the page loaded) and "alive" minute 1 (stayed to play).
            bump(data.setdefault("sail", {}).setdefault(today(), {}), s(r, "v"))
        elif kind == "goal":
            # A mission or the daily chest done: how far down the Captain's Log players get.
            bump(data.setdefault("goals", {}).setdefault(today(), {}), s(r, "g", 24), most=80)
        elif kind == "fps":
            data["fps"].append({k: r.get(k) for k in ("fps", "slow", "q", "gpu", "dev", "mob", "mode", "w", "h")} | {"day": today()})
            del data["fps"][:-500]
        elif kind in ("error", "js"):
            msg = s(r, "msg", 1500)  # a crash's stack lives in its message
            sig = kind + ":" + re.sub(r"\d+", "#", msg)[:200]
            e = data["errors"].get(sig)
            if e is None:
                if len(data["errors"]) >= MAX_ERRORS:
                    return
                e = data["errors"][sig] = {"kind": kind, "type": s(r, "type", 20), "msg": msg, "stack": s(r, "stack", 1500),
                                           "count": 0, "first": today(), "last": "", "gpu": {}, "dev": {}, "q": {}, "mode": {}, "v": {}}
            e["count"] += 1
            e["last"] = today()
            for k in ("gpu", "dev", "q", "mode", "v"):
                bump(e[k], s(r, k))
            samples = e.setdefault("samples", [])
            samples.append({"day": today(), "s": s(r, "s", 12), "t": num(r, "t"), "mem": num(r, "mem"), "heap": num(r, "heap"),
                            "q": s(r, "q", 8), "mob": bool(r.get("mob")), "v": s(r, "v", 10), "gpu": s(r, "gpu", 60)})
            del samples[:-MAX_SAMPLES]
            if FATAL.search(msg) and r.get("s"):
                ids = data.setdefault("crashed", {}).setdefault(today(), {}).setdefault(s(r, "v"), [])
                if s(r, "s", 12) not in ids and len(ids) < MAX_CRASHED:
                    ids.append(s(r, "s", 12))
        else:
            return
        dirty = True


class Handler(BaseHTTPRequestHandler):
    def log_message(self, *_):
        pass

    def reply(self, code, body=b"", kind="text/plain"):
        self.send_response(code)
        self.send_header("Access-Control-Allow-Origin", "*")
        self.send_header("Access-Control-Allow-Headers", "Content-Type")
        self.send_header("Content-Type", kind)
        self.send_header("Content-Length", str(len(body)))
        self.end_headers()
        self.wfile.write(body)

    def do_OPTIONS(self):
        self.reply(204)

    def do_POST(self):
        if urlparse(self.path).path != "/cos2/report":
            return self.reply(404)
        n = int(self.headers.get("Content-Length") or 0)
        if n <= 0 or n > MAX_BODY:
            return self.reply(413)
        body = self.rfile.read(n)
        who = (self.headers.get("X-Forwarded-For") or self.client_address[0]).split(",")[0].strip()
        now = time.time()
        with lock:
            q = recent.setdefault(who, deque())
            while q and now - q[0] > 600:
                q.popleft()
            if len(q) >= PER_IP:
                return self.reply(429)
            q.append(now)
            if len(recent) > 5000:
                recent.clear()
        try:
            r = json.loads(body)
            if isinstance(r, dict):
                record(r)
        except Exception:
            return self.reply(400)
        self.reply(204)

    def do_GET(self):
        u = urlparse(self.path)
        if u.path != "/cos2/errors" or not KEY or parse_qs(u.query).get("key", [""])[0] != KEY:
            return self.reply(404)
        with lock:
            body = json.dumps(data, indent=1).encode()
        self.reply(200, body, "application/json")


def saver():
    while True:
        time.sleep(30)
        try:
            save()
        except Exception as e:
            print("[reports] save failed: %s" % e, flush=True)


def stop(*_):
    save()
    sys.exit(0)


if __name__ == "__main__":
    signal.signal(signal.SIGTERM, stop)
    signal.signal(signal.SIGINT, stop)
    threading.Thread(target=saver, daemon=True).start()
    print("[reports] listening on %d (%d errors so far)" % (PORT, len(data["errors"])), flush=True)
    ThreadingHTTPServer(("127.0.0.1", PORT), Handler).serve_forever()