Spaces:
Running
Running
main - Give workers stable names instead of raw pids
Browse filesPids change on every restart, making it hard to tell workers apart across
deploys or to notice a specific one crash-looping. Each worker now claims
a name ("Worker A", "Worker B", ...) from a small shared file the first
time it starts, keeps it for its whole lifetime, and never reuses a name
for a different pid — so a restart shows up as a new name appearing while
the old one's alert fades out. Alerts and worker cards use the name; the
pid is still available as a tooltip on the card.
- main.py +1 -1
- similarity.py +44 -3
main.py
CHANGED
|
@@ -113,7 +113,7 @@ function render(d){
|
|
| 113 |
document.getElementById('grid').innerHTML=w.map(function(x){
|
| 114 |
var c=workerCol(x.rss_mb),p=(x.rss_mb/maxRss*100).toFixed(0);
|
| 115 |
return '<div class="card" style="--c:'+c+'">'+
|
| 116 |
-
'<div class="card-top"><span class="pid">pid '+x.pid+'</span><span class="cn">'+x.cache_size+'/'+MAX+'</span></div>'+
|
| 117 |
'<div class="rss">'+x.rss_mb+'</div>'+
|
| 118 |
'<div class="card-bot"><span class="unit">MB</span><span class="req">'+((x.requests||0)+' req')+'</span></div>'+
|
| 119 |
'<div class="track"><div class="fill" style="width:'+p+'%;background:'+c+'"></div></div>'+
|
|
|
|
| 113 |
document.getElementById('grid').innerHTML=w.map(function(x){
|
| 114 |
var c=workerCol(x.rss_mb),p=(x.rss_mb/maxRss*100).toFixed(0);
|
| 115 |
return '<div class="card" style="--c:'+c+'">'+
|
| 116 |
+
'<div class="card-top"><span class="pid" title="pid '+x.pid+'">'+(x.name||('pid '+x.pid))+'</span><span class="cn">'+x.cache_size+'/'+MAX+'</span></div>'+
|
| 117 |
'<div class="rss">'+x.rss_mb+'</div>'+
|
| 118 |
'<div class="card-bot"><span class="unit">MB</span><span class="req">'+((x.requests||0)+' req')+'</span></div>'+
|
| 119 |
'<div class="track"><div class="fill" style="width:'+p+'%;background:'+c+'"></div></div>'+
|
similarity.py
CHANGED
|
@@ -163,6 +163,42 @@ def _update_cpu_percent():
|
|
| 163 |
_MEM_STATS_DIR = "/tmp/similarity_mem_stats"
|
| 164 |
os.makedirs(_MEM_STATS_DIR, exist_ok=True)
|
| 165 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 166 |
|
| 167 |
def _read_cgroup_memory():
|
| 168 |
# cgroup v2 (modern Docker)
|
|
@@ -219,6 +255,7 @@ def get_memory_stats():
|
|
| 219 |
|
| 220 |
stats = {
|
| 221 |
"pid": os.getpid(),
|
|
|
|
| 222 |
"rss_mb": rss_mb,
|
| 223 |
"cache_size": cache_size,
|
| 224 |
"requests": _request_count,
|
|
@@ -259,6 +296,8 @@ def start_memory_trimmer(interval_seconds=300):
|
|
| 259 |
def _read_all_worker_files():
|
| 260 |
entries = []
|
| 261 |
for filename in os.listdir(_MEM_STATS_DIR):
|
|
|
|
|
|
|
| 262 |
fpath = os.path.join(_MEM_STATS_DIR, filename)
|
| 263 |
try:
|
| 264 |
with open(fpath) as f:
|
|
@@ -284,18 +323,20 @@ def get_dashboard_data():
|
|
| 284 |
pass
|
| 285 |
continue
|
| 286 |
|
|
|
|
|
|
|
| 287 |
if age > _STALE_AFTER_SECONDS:
|
| 288 |
-
alerts.append(f"
|
| 289 |
continue
|
| 290 |
|
| 291 |
alive.append(stats)
|
| 292 |
|
| 293 |
if (stats.get("uptime_seconds", 0) > _RSS_SPIKE_MIN_UPTIME_SECONDS
|
| 294 |
and stats.get("rss_delta_2min_mb", 0) > _RSS_SPIKE_MB):
|
| 295 |
-
alerts.append(f"
|
| 296 |
|
| 297 |
if stats.get("evictions_per_min", 0) > _CACHE_THRASH_EVICTIONS_PER_MIN:
|
| 298 |
-
alerts.append(f"
|
| 299 |
|
| 300 |
alive.sort(key=lambda s: s["pid"])
|
| 301 |
|
|
|
|
| 163 |
_MEM_STATS_DIR = "/tmp/similarity_mem_stats"
|
| 164 |
os.makedirs(_MEM_STATS_DIR, exist_ok=True)
|
| 165 |
|
| 166 |
+
_NAMES_FILE = os.path.join(_MEM_STATS_DIR, "_names.json")
|
| 167 |
+
_WORKER_NAME_ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
def _assign_worker_name():
|
| 171 |
+
try:
|
| 172 |
+
with open(_NAMES_FILE) as f:
|
| 173 |
+
names = json.load(f)
|
| 174 |
+
except (OSError, json.JSONDecodeError):
|
| 175 |
+
names = {}
|
| 176 |
+
|
| 177 |
+
pid_key = str(os.getpid())
|
| 178 |
+
if pid_key in names:
|
| 179 |
+
return names[pid_key]
|
| 180 |
+
|
| 181 |
+
used = set(names.values())
|
| 182 |
+
for letter in _WORKER_NAME_ALPHABET:
|
| 183 |
+
candidate = f"Worker {letter}"
|
| 184 |
+
if candidate not in used:
|
| 185 |
+
break
|
| 186 |
+
else:
|
| 187 |
+
candidate = f"Worker {len(names) + 1}"
|
| 188 |
+
|
| 189 |
+
names[pid_key] = candidate
|
| 190 |
+
try:
|
| 191 |
+
with open(_NAMES_FILE, "w") as f:
|
| 192 |
+
json.dump(names, f)
|
| 193 |
+
except OSError:
|
| 194 |
+
pass
|
| 195 |
+
return candidate
|
| 196 |
+
|
| 197 |
+
|
| 198 |
+
# Assigned once at import time and kept for the process's lifetime — a name is
|
| 199 |
+
# never reused for a different pid, so a restart is visible as a new name.
|
| 200 |
+
_worker_name = _assign_worker_name()
|
| 201 |
+
|
| 202 |
|
| 203 |
def _read_cgroup_memory():
|
| 204 |
# cgroup v2 (modern Docker)
|
|
|
|
| 255 |
|
| 256 |
stats = {
|
| 257 |
"pid": os.getpid(),
|
| 258 |
+
"name": _worker_name,
|
| 259 |
"rss_mb": rss_mb,
|
| 260 |
"cache_size": cache_size,
|
| 261 |
"requests": _request_count,
|
|
|
|
| 296 |
def _read_all_worker_files():
|
| 297 |
entries = []
|
| 298 |
for filename in os.listdir(_MEM_STATS_DIR):
|
| 299 |
+
if filename.startswith("_"):
|
| 300 |
+
continue # internal files (e.g. _names.json), not per-worker stats
|
| 301 |
fpath = os.path.join(_MEM_STATS_DIR, filename)
|
| 302 |
try:
|
| 303 |
with open(fpath) as f:
|
|
|
|
| 323 |
pass
|
| 324 |
continue
|
| 325 |
|
| 326 |
+
name = stats.get("name") or f"pid {stats['pid']}"
|
| 327 |
+
|
| 328 |
if age > _STALE_AFTER_SECONDS:
|
| 329 |
+
alerts.append(f"{name} sem reportar há {int(age)}s")
|
| 330 |
continue
|
| 331 |
|
| 332 |
alive.append(stats)
|
| 333 |
|
| 334 |
if (stats.get("uptime_seconds", 0) > _RSS_SPIKE_MIN_UPTIME_SECONDS
|
| 335 |
and stats.get("rss_delta_2min_mb", 0) > _RSS_SPIKE_MB):
|
| 336 |
+
alerts.append(f"{name} RAM subiu {stats['rss_delta_2min_mb']:.0f} MB em 2 min")
|
| 337 |
|
| 338 |
if stats.get("evictions_per_min", 0) > _CACHE_THRASH_EVICTIONS_PER_MIN:
|
| 339 |
+
alerts.append(f"{name} cache a saturar (~{stats['evictions_per_min']:.0f} evicções/min)")
|
| 340 |
|
| 341 |
alive.sort(key=lambda s: s["pid"])
|
| 342 |
|