Nottybro commited on
Commit
d13db56
·
verified ·
1 Parent(s): 9df8967

Reports: crash timing, device memory, heap, per-build sessions and crashes, play-time pings

Browse files
Files changed (1) hide show
  1. reports.py +215 -176
reports.py CHANGED
@@ -1,176 +1,215 @@
1
- """
2
- Captain of Sea error reports. The game posts what broke in a player's browser (Assets/Game/Scripts/
3
- ErrorReporter.cs: no names, ids or emails), and this keeps a tally per distinct error, plus how many
4
- sessions started and the frame rates players got, in <CAPTAIN_SAVES>/reports/errors.json (backed up
5
- with the saves by run.py). nginx sends /cos2/ here.
6
-
7
- POST /cos2/report one report (JSON, sent as text/plain so browsers skip the CORS preflight)
8
- GET /cos2/errors?key=... the tally (key = the REPORT_KEY secret; off without it)
9
- """
10
- import json
11
- import os
12
- import re
13
- import signal
14
- import sys
15
- import threading
16
- import time
17
- from collections import deque
18
- from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
19
- from urllib.parse import parse_qs, urlparse
20
-
21
- PATH = os.path.join(os.environ.get("CAPTAIN_SAVES", "/home/user/saves"), "reports", "errors.json")
22
- KEY = os.environ.get("REPORT_KEY", "")
23
- PORT = int(os.environ.get("REPORTS_PORT", "8001"))
24
- MAX_BODY = 8192
25
- MAX_ERRORS = 300 # distinct errors kept
26
- MAX_KEYS = 20 # per breakdown (gpu, browser...); the rest count as "other"
27
- PER_IP = 60 # reports per address per 10 minutes
28
-
29
- lock = threading.Lock()
30
- dirty = False
31
- recent = {} # address -> deque of times (memory only, never saved)
32
-
33
-
34
- def today():
35
- return time.strftime("%Y-%m-%d", time.gmtime())
36
-
37
-
38
- def load():
39
- try:
40
- with open(PATH, encoding="utf-8") as f:
41
- return json.load(f)
42
- except Exception:
43
- return {"since": today(), "sessions": {}, "fps": [], "errors": {}}
44
-
45
-
46
- data = load()
47
-
48
-
49
- def save():
50
- global dirty
51
- with lock:
52
- if not dirty:
53
- return
54
- text = json.dumps(data, separators=(",", ":"))
55
- dirty = False
56
- os.makedirs(os.path.dirname(PATH), exist_ok=True)
57
- tmp = PATH + ".tmp"
58
- with open(tmp, "w", encoding="utf-8") as f:
59
- f.write(text)
60
- os.replace(tmp, PATH)
61
-
62
-
63
- def bump(d, key, n=1):
64
- key = str(key or "?")[:80]
65
- if key not in d and len(d) >= MAX_KEYS:
66
- key = "other"
67
- d[key] = d.get(key, 0) + n
68
-
69
-
70
- def s(r, k, n=120):
71
- v = r.get(k)
72
- return str(v)[:n] if v is not None else ""
73
-
74
-
75
- def record(r):
76
- global dirty
77
- kind = s(r, "kind", 12)
78
- with lock:
79
- if kind == "hello":
80
- day = data["sessions"].setdefault(today(), {"n": 0, "mobile": 0, "q": {}, "gpu": {}, "dev": {}, "os": {}})
81
- day["n"] += 1
82
- day["mobile"] += 1 if r.get("mob") else 0
83
- for k in ("q", "gpu", "dev", "os"):
84
- bump(day[k], s(r, k))
85
- elif kind == "fps":
86
- data["fps"].append({k: r.get(k) for k in ("fps", "slow", "q", "gpu", "dev", "mob", "mode", "w", "h")} | {"day": today()})
87
- del data["fps"][:-500]
88
- elif kind in ("error", "js"):
89
- msg = s(r, "msg", 400)
90
- sig = kind + ":" + re.sub(r"\d+", "#", msg)[:200]
91
- e = data["errors"].get(sig)
92
- if e is None:
93
- if len(data["errors"]) >= MAX_ERRORS:
94
- return
95
- e = data["errors"][sig] = {"kind": kind, "type": s(r, "type", 20), "msg": msg, "stack": s(r, "stack", 1500),
96
- "count": 0, "first": today(), "last": "", "gpu": {}, "dev": {}, "q": {}, "mode": {}, "v": {}}
97
- e["count"] += 1
98
- e["last"] = today()
99
- for k in ("gpu", "dev", "q", "mode", "v"):
100
- bump(e[k], s(r, k))
101
- else:
102
- return
103
- dirty = True
104
-
105
-
106
- class Handler(BaseHTTPRequestHandler):
107
- def log_message(self, *_):
108
- pass
109
-
110
- def reply(self, code, body=b"", kind="text/plain"):
111
- self.send_response(code)
112
- self.send_header("Access-Control-Allow-Origin", "*")
113
- self.send_header("Access-Control-Allow-Headers", "Content-Type")
114
- self.send_header("Content-Type", kind)
115
- self.send_header("Content-Length", str(len(body)))
116
- self.end_headers()
117
- self.wfile.write(body)
118
-
119
- def do_OPTIONS(self):
120
- self.reply(204)
121
-
122
- def do_POST(self):
123
- if urlparse(self.path).path != "/cos2/report":
124
- return self.reply(404)
125
- n = int(self.headers.get("Content-Length") or 0)
126
- if n <= 0 or n > MAX_BODY:
127
- return self.reply(413)
128
- body = self.rfile.read(n)
129
- who = (self.headers.get("X-Forwarded-For") or self.client_address[0]).split(",")[0].strip()
130
- now = time.time()
131
- with lock:
132
- q = recent.setdefault(who, deque())
133
- while q and now - q[0] > 600:
134
- q.popleft()
135
- if len(q) >= PER_IP:
136
- return self.reply(429)
137
- q.append(now)
138
- if len(recent) > 5000:
139
- recent.clear()
140
- try:
141
- r = json.loads(body)
142
- if isinstance(r, dict):
143
- record(r)
144
- except Exception:
145
- return self.reply(400)
146
- self.reply(204)
147
-
148
- def do_GET(self):
149
- u = urlparse(self.path)
150
- if u.path != "/cos2/errors" or not KEY or parse_qs(u.query).get("key", [""])[0] != KEY:
151
- return self.reply(404)
152
- with lock:
153
- body = json.dumps(data, indent=1).encode()
154
- self.reply(200, body, "application/json")
155
-
156
-
157
- def saver():
158
- while True:
159
- time.sleep(30)
160
- try:
161
- save()
162
- except Exception as e:
163
- print("[reports] save failed: %s" % e, flush=True)
164
-
165
-
166
- def stop(*_):
167
- save()
168
- sys.exit(0)
169
-
170
-
171
- if __name__ == "__main__":
172
- signal.signal(signal.SIGTERM, stop)
173
- signal.signal(signal.SIGINT, stop)
174
- threading.Thread(target=saver, daemon=True).start()
175
- print("[reports] listening on %d (%d errors so far)" % (PORT, len(data["errors"])), flush=True)
176
- ThreadingHTTPServer(("127.0.0.1", PORT), Handler).serve_forever()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Captain of Sea error reports. The game posts what broke in a player's browser (Assets/Game/Scripts/
3
+ ErrorReporter.cs: no names, ids or emails), and this keeps a tally per distinct error, plus how many
4
+ sessions started and the frame rates players got, in <CAPTAIN_SAVES>/reports/errors.json (backed up
5
+ with the saves by run.py). nginx sends /cos2/ here.
6
+
7
+ Crashes keep their last few reports in full (when in the session, the device's memory, the heap), so a
8
+ crash can be placed: at load, or minutes into a voyage. Sessions are counted per build, and the sessions
9
+ a fatal error ended per build too, so each build's crash rate reads straight off. "alive" pings (minutes
10
+ into a voyage, with the heap) show how long players stay and whether the heap creeps.
11
+
12
+ POST /cos2/report one report (JSON, sent as text/plain so browsers skip the CORS preflight)
13
+ GET /cos2/errors?key=... the tally (key = the REPORT_KEY secret; off without it)
14
+ """
15
+ import json
16
+ import os
17
+ import re
18
+ import signal
19
+ import sys
20
+ import threading
21
+ import time
22
+ from collections import deque
23
+ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
24
+ from urllib.parse import parse_qs, urlparse
25
+
26
+ PATH = os.path.join(os.environ.get("CAPTAIN_SAVES", "/home/user/saves"), "reports", "errors.json")
27
+ KEY = os.environ.get("REPORT_KEY", "")
28
+ PORT = int(os.environ.get("REPORTS_PORT", "8001"))
29
+ MAX_BODY = 8192
30
+ MAX_ERRORS = 300 # distinct errors kept
31
+ MAX_KEYS = 20 # per breakdown (gpu, browser...); the rest count as "other"
32
+ MAX_SAMPLES = 40 # full reports kept per error
33
+ MAX_CRASHED = 3000 # crashed session ids kept per day
34
+ FATAL = re.compile(r"OOM|Could not allocate|out of bounds|unreachable|context lost|Aborted", re.I)
35
+ PER_IP = 60 # reports per address per 10 minutes
36
+
37
+ lock = threading.Lock()
38
+ dirty = False
39
+ recent = {} # address -> deque of times (memory only, never saved)
40
+
41
+
42
+ def today():
43
+ return time.strftime("%Y-%m-%d", time.gmtime())
44
+
45
+
46
+ def load():
47
+ try:
48
+ with open(PATH, encoding="utf-8") as f:
49
+ return json.load(f)
50
+ except Exception:
51
+ return {"since": today(), "sessions": {}, "fps": [], "errors": {}}
52
+
53
+
54
+ def num(r, k):
55
+ v = r.get(k)
56
+ return round(v) if isinstance(v, (int, float)) and not isinstance(v, bool) else None
57
+
58
+
59
+ data = load()
60
+
61
+
62
+ def save():
63
+ global dirty
64
+ with lock:
65
+ if not dirty:
66
+ return
67
+ text = json.dumps(data, separators=(",", ":"))
68
+ dirty = False
69
+ os.makedirs(os.path.dirname(PATH), exist_ok=True)
70
+ tmp = PATH + ".tmp"
71
+ with open(tmp, "w", encoding="utf-8") as f:
72
+ f.write(text)
73
+ os.replace(tmp, PATH)
74
+
75
+
76
+ def bump(d, key, n=1):
77
+ key = str(key or "?")[:80]
78
+ if key not in d and len(d) >= MAX_KEYS:
79
+ key = "other"
80
+ d[key] = d.get(key, 0) + n
81
+
82
+
83
+ def memband(r):
84
+ """The device's memory, in bands (the browser rounds it anyway)."""
85
+ m = num(r, "mem")
86
+ if not m:
87
+ return "?"
88
+ for top in (1024, 2048, 3072, 4096, 6144, 8192, 16384):
89
+ if m <= top:
90
+ return "<=%dG" % (top // 1024)
91
+ return ">16G"
92
+
93
+
94
+ def s(r, k, n=120):
95
+ v = r.get(k)
96
+ return str(v)[:n] if v is not None else ""
97
+
98
+
99
+ def record(r):
100
+ global dirty
101
+ kind = s(r, "kind", 12)
102
+ with lock:
103
+ if kind == "hello":
104
+ day = data["sessions"].setdefault(today(), {"n": 0, "mobile": 0, "q": {}, "gpu": {}, "dev": {}, "os": {}})
105
+ day["n"] += 1
106
+ day["mobile"] += 1 if r.get("mob") else 0
107
+ for k in ("q", "gpu", "dev", "os", "v", "mem"):
108
+ bump(day.setdefault(k, {}), memband(r) if k == "mem" else s(r, k))
109
+ elif kind == "alive":
110
+ # A ping at minute 1, 3, 5, 10, 20, 30... of a voyage: how long players stay, and the heap then.
111
+ day = data.setdefault("alive", {}).setdefault(today(), {})
112
+ bump(day.setdefault(s(r, "v"), {}), num(r, "min"))
113
+ data.setdefault("heap", []).append({k: r.get(k) for k in ("min", "heap", "q", "mob", "mode", "mem", "v")}
114
+ | {"gpu": s(r, "gpu", 60), "day": today()})
115
+ del data["heap"][:-800]
116
+ elif kind == "fps":
117
+ data["fps"].append({k: r.get(k) for k in ("fps", "slow", "q", "gpu", "dev", "mob", "mode", "w", "h")} | {"day": today()})
118
+ del data["fps"][:-500]
119
+ elif kind in ("error", "js"):
120
+ msg = s(r, "msg", 1500) # a crash's stack lives in its message
121
+ sig = kind + ":" + re.sub(r"\d+", "#", msg)[:200]
122
+ e = data["errors"].get(sig)
123
+ if e is None:
124
+ if len(data["errors"]) >= MAX_ERRORS:
125
+ return
126
+ e = data["errors"][sig] = {"kind": kind, "type": s(r, "type", 20), "msg": msg, "stack": s(r, "stack", 1500),
127
+ "count": 0, "first": today(), "last": "", "gpu": {}, "dev": {}, "q": {}, "mode": {}, "v": {}}
128
+ e["count"] += 1
129
+ e["last"] = today()
130
+ for k in ("gpu", "dev", "q", "mode", "v"):
131
+ bump(e[k], s(r, k))
132
+ samples = e.setdefault("samples", [])
133
+ samples.append({"day": today(), "s": s(r, "s", 12), "t": num(r, "t"), "mem": num(r, "mem"), "heap": num(r, "heap"),
134
+ "q": s(r, "q", 8), "mob": bool(r.get("mob")), "v": s(r, "v", 10), "gpu": s(r, "gpu", 60)})
135
+ del samples[:-MAX_SAMPLES]
136
+ if FATAL.search(msg) and r.get("s"):
137
+ ids = data.setdefault("crashed", {}).setdefault(today(), {}).setdefault(s(r, "v"), [])
138
+ if s(r, "s", 12) not in ids and len(ids) < MAX_CRASHED:
139
+ ids.append(s(r, "s", 12))
140
+ else:
141
+ return
142
+ dirty = True
143
+
144
+
145
+ class Handler(BaseHTTPRequestHandler):
146
+ def log_message(self, *_):
147
+ pass
148
+
149
+ def reply(self, code, body=b"", kind="text/plain"):
150
+ self.send_response(code)
151
+ self.send_header("Access-Control-Allow-Origin", "*")
152
+ self.send_header("Access-Control-Allow-Headers", "Content-Type")
153
+ self.send_header("Content-Type", kind)
154
+ self.send_header("Content-Length", str(len(body)))
155
+ self.end_headers()
156
+ self.wfile.write(body)
157
+
158
+ def do_OPTIONS(self):
159
+ self.reply(204)
160
+
161
+ def do_POST(self):
162
+ if urlparse(self.path).path != "/cos2/report":
163
+ return self.reply(404)
164
+ n = int(self.headers.get("Content-Length") or 0)
165
+ if n <= 0 or n > MAX_BODY:
166
+ return self.reply(413)
167
+ body = self.rfile.read(n)
168
+ who = (self.headers.get("X-Forwarded-For") or self.client_address[0]).split(",")[0].strip()
169
+ now = time.time()
170
+ with lock:
171
+ q = recent.setdefault(who, deque())
172
+ while q and now - q[0] > 600:
173
+ q.popleft()
174
+ if len(q) >= PER_IP:
175
+ return self.reply(429)
176
+ q.append(now)
177
+ if len(recent) > 5000:
178
+ recent.clear()
179
+ try:
180
+ r = json.loads(body)
181
+ if isinstance(r, dict):
182
+ record(r)
183
+ except Exception:
184
+ return self.reply(400)
185
+ self.reply(204)
186
+
187
+ def do_GET(self):
188
+ u = urlparse(self.path)
189
+ if u.path != "/cos2/errors" or not KEY or parse_qs(u.query).get("key", [""])[0] != KEY:
190
+ return self.reply(404)
191
+ with lock:
192
+ body = json.dumps(data, indent=1).encode()
193
+ self.reply(200, body, "application/json")
194
+
195
+
196
+ def saver():
197
+ while True:
198
+ time.sleep(30)
199
+ try:
200
+ save()
201
+ except Exception as e:
202
+ print("[reports] save failed: %s" % e, flush=True)
203
+
204
+
205
+ def stop(*_):
206
+ save()
207
+ sys.exit(0)
208
+
209
+
210
+ if __name__ == "__main__":
211
+ signal.signal(signal.SIGTERM, stop)
212
+ signal.signal(signal.SIGINT, stop)
213
+ threading.Thread(target=saver, daemon=True).start()
214
+ print("[reports] listening on %d (%d errors so far)" % (PORT, len(data["errors"])), flush=True)
215
+ ThreadingHTTPServer(("127.0.0.1", PORT), Handler).serve_forever()