File size: 8,677 Bytes
031a83d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 | """
Captain of Sea error reports. The game posts what broke in a player's browser (Assets/Game/Scripts/
ErrorReporter.cs: no names, ids or emails), and this keeps a tally per distinct error, plus how many
sessions started and the frame rates players got, in <CAPTAIN_SAVES>/reports/errors.json (backed up
with the saves by run.py). nginx sends /cos2/ here.
Crashes keep their last few reports in full (when in the session, the device's memory, the heap), so a
crash can be placed: at load, or minutes into a voyage. Sessions are counted per build, and the sessions
a fatal error ended per build too, so each build's crash rate reads straight off. "alive" pings (minutes
into a voyage, with the heap) show how long players stay and whether the heap creeps. "sail" (got aboard)
sits between a session's "hello" and its first "alive": the drop-off at each step. "goal" counts missions
done by id, so how far down the Captain's Log players get reads straight off too.
POST /cos2/report one report (JSON, sent as text/plain so browsers skip the CORS preflight)
GET /cos2/errors?key=... the tally (key = the REPORT_KEY secret; off without it)
"""
import json
import os
import re
import signal
import sys
import threading
import time
from collections import deque
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from urllib.parse import parse_qs, urlparse
PATH = os.path.join(os.environ.get("CAPTAIN_SAVES", "/home/user/saves"), "reports", "errors.json")
KEY = os.environ.get("REPORT_KEY", "")
PORT = int(os.environ.get("REPORTS_PORT", "8001"))
MAX_BODY = 8192
MAX_ERRORS = 300 # distinct errors kept
MAX_KEYS = 20 # per breakdown (gpu, browser...); the rest count as "other"
MAX_SAMPLES = 40 # full reports kept per error
MAX_CRASHED = 3000 # crashed session ids kept per day
FATAL = re.compile(r"OOM|Could not allocate|out of bounds|unreachable|context lost|Aborted", re.I)
PER_IP = 60 # reports per address per 10 minutes
lock = threading.Lock()
dirty = False
recent = {} # address -> deque of times (memory only, never saved)
def today():
return time.strftime("%Y-%m-%d", time.gmtime())
def load():
try:
with open(PATH, encoding="utf-8") as f:
return json.load(f)
except Exception:
return {"since": today(), "sessions": {}, "fps": [], "errors": {}}
def num(r, k):
v = r.get(k)
return round(v) if isinstance(v, (int, float)) and not isinstance(v, bool) else None
data = load()
def save():
global dirty
with lock:
if not dirty:
return
text = json.dumps(data, separators=(",", ":"))
dirty = False
os.makedirs(os.path.dirname(PATH), exist_ok=True)
tmp = PATH + ".tmp"
with open(tmp, "w", encoding="utf-8") as f:
f.write(text)
os.replace(tmp, PATH)
def bump(d, key, n=1, most=MAX_KEYS):
key = str(key or "?")[:80]
if key not in d and len(d) >= most:
key = "other"
d[key] = d.get(key, 0) + n
def memband(r):
"""The device's memory, in bands (the browser rounds it anyway)."""
m = num(r, "mem")
if not m:
return "?"
for top in (1024, 2048, 3072, 4096, 6144, 8192, 16384):
if m <= top:
return "<=%dG" % (top // 1024)
return ">16G"
def s(r, k, n=120):
v = r.get(k)
return str(v)[:n] if v is not None else ""
def record(r):
global dirty
kind = s(r, "kind", 12)
with lock:
if kind == "hello":
day = data["sessions"].setdefault(today(), {"n": 0, "mobile": 0, "q": {}, "gpu": {}, "dev": {}, "os": {}})
day["n"] += 1
day["mobile"] += 1 if r.get("mob") else 0
for k in ("q", "gpu", "dev", "os", "v", "mem"):
bump(day.setdefault(k, {}), memband(r) if k == "mem" else s(r, k))
elif kind == "alive":
# A ping at minute 1, 3, 5, 10, 20, 30... of a voyage: how long players stay, and the heap then.
day = data.setdefault("alive", {}).setdefault(today(), {})
bump(day.setdefault(s(r, "v"), {}), num(r, "min"))
data.setdefault("heap", []).append({k: r.get(k) for k in ("min", "heap", "q", "mob", "mode", "mem", "v")}
| {"gpu": s(r, "gpu", 60), "day": today()})
del data["heap"][:-800]
elif kind == "sail":
# Got aboard a ship: between "hello" (the page loaded) and "alive" minute 1 (stayed to play).
bump(data.setdefault("sail", {}).setdefault(today(), {}), s(r, "v"))
elif kind == "goal":
# A mission or the daily chest done: how far down the Captain's Log players get.
bump(data.setdefault("goals", {}).setdefault(today(), {}), s(r, "g", 24), most=80)
elif kind == "fps":
data["fps"].append({k: r.get(k) for k in ("fps", "slow", "q", "gpu", "dev", "mob", "mode", "w", "h")} | {"day": today()})
del data["fps"][:-500]
elif kind in ("error", "js"):
msg = s(r, "msg", 1500) # a crash's stack lives in its message
sig = kind + ":" + re.sub(r"\d+", "#", msg)[:200]
e = data["errors"].get(sig)
if e is None:
if len(data["errors"]) >= MAX_ERRORS:
return
e = data["errors"][sig] = {"kind": kind, "type": s(r, "type", 20), "msg": msg, "stack": s(r, "stack", 1500),
"count": 0, "first": today(), "last": "", "gpu": {}, "dev": {}, "q": {}, "mode": {}, "v": {}}
e["count"] += 1
e["last"] = today()
for k in ("gpu", "dev", "q", "mode", "v"):
bump(e[k], s(r, k))
samples = e.setdefault("samples", [])
samples.append({"day": today(), "s": s(r, "s", 12), "t": num(r, "t"), "mem": num(r, "mem"), "heap": num(r, "heap"),
"q": s(r, "q", 8), "mob": bool(r.get("mob")), "v": s(r, "v", 10), "gpu": s(r, "gpu", 60)})
del samples[:-MAX_SAMPLES]
if FATAL.search(msg) and r.get("s"):
ids = data.setdefault("crashed", {}).setdefault(today(), {}).setdefault(s(r, "v"), [])
if s(r, "s", 12) not in ids and len(ids) < MAX_CRASHED:
ids.append(s(r, "s", 12))
else:
return
dirty = True
class Handler(BaseHTTPRequestHandler):
def log_message(self, *_):
pass
def reply(self, code, body=b"", kind="text/plain"):
self.send_response(code)
self.send_header("Access-Control-Allow-Origin", "*")
self.send_header("Access-Control-Allow-Headers", "Content-Type")
self.send_header("Content-Type", kind)
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def do_OPTIONS(self):
self.reply(204)
def do_POST(self):
if urlparse(self.path).path != "/cos2/report":
return self.reply(404)
n = int(self.headers.get("Content-Length") or 0)
if n <= 0 or n > MAX_BODY:
return self.reply(413)
body = self.rfile.read(n)
who = (self.headers.get("X-Forwarded-For") or self.client_address[0]).split(",")[0].strip()
now = time.time()
with lock:
q = recent.setdefault(who, deque())
while q and now - q[0] > 600:
q.popleft()
if len(q) >= PER_IP:
return self.reply(429)
q.append(now)
if len(recent) > 5000:
recent.clear()
try:
r = json.loads(body)
if isinstance(r, dict):
record(r)
except Exception:
return self.reply(400)
self.reply(204)
def do_GET(self):
u = urlparse(self.path)
if u.path != "/cos2/errors" or not KEY or parse_qs(u.query).get("key", [""])[0] != KEY:
return self.reply(404)
with lock:
body = json.dumps(data, indent=1).encode()
self.reply(200, body, "application/json")
def saver():
while True:
time.sleep(30)
try:
save()
except Exception as e:
print("[reports] save failed: %s" % e, flush=True)
def stop(*_):
save()
sys.exit(0)
if __name__ == "__main__":
signal.signal(signal.SIGTERM, stop)
signal.signal(signal.SIGINT, stop)
threading.Thread(target=saver, daemon=True).start()
print("[reports] listening on %d (%d errors so far)" % (PORT, len(data["errors"])), flush=True)
ThreadingHTTPServer(("127.0.0.1", PORT), Handler).serve_forever()
|