lpedro commited on
Commit
46ea438
·
1 Parent(s): 3200602

main - Give workers stable names instead of raw pids

Browse files

Pids change on every restart, making it hard to tell workers apart across
deploys or to notice a specific one crash-looping. Each worker now claims
a name ("Worker A", "Worker B", ...) from a small shared file the first
time it starts, keeps it for its whole lifetime, and never reuses a name
for a different pid — so a restart shows up as a new name appearing while
the old one's alert fades out. Alerts and worker cards use the name; the
pid is still available as a tooltip on the card.

Files changed (2) hide show
  1. main.py +1 -1
  2. similarity.py +44 -3
main.py CHANGED
@@ -113,7 +113,7 @@ function render(d){
113
  document.getElementById('grid').innerHTML=w.map(function(x){
114
  var c=workerCol(x.rss_mb),p=(x.rss_mb/maxRss*100).toFixed(0);
115
  return '<div class="card" style="--c:'+c+'">'+
116
- '<div class="card-top"><span class="pid">pid '+x.pid+'</span><span class="cn">'+x.cache_size+'/'+MAX+'</span></div>'+
117
  '<div class="rss">'+x.rss_mb+'</div>'+
118
  '<div class="card-bot"><span class="unit">MB</span><span class="req">'+((x.requests||0)+' req')+'</span></div>'+
119
  '<div class="track"><div class="fill" style="width:'+p+'%;background:'+c+'"></div></div>'+
 
113
  document.getElementById('grid').innerHTML=w.map(function(x){
114
  var c=workerCol(x.rss_mb),p=(x.rss_mb/maxRss*100).toFixed(0);
115
  return '<div class="card" style="--c:'+c+'">'+
116
+ '<div class="card-top"><span class="pid" title="pid '+x.pid+'">'+(x.name||('pid '+x.pid))+'</span><span class="cn">'+x.cache_size+'/'+MAX+'</span></div>'+
117
  '<div class="rss">'+x.rss_mb+'</div>'+
118
  '<div class="card-bot"><span class="unit">MB</span><span class="req">'+((x.requests||0)+' req')+'</span></div>'+
119
  '<div class="track"><div class="fill" style="width:'+p+'%;background:'+c+'"></div></div>'+
similarity.py CHANGED
@@ -163,6 +163,42 @@ def _update_cpu_percent():
163
  _MEM_STATS_DIR = "/tmp/similarity_mem_stats"
164
  os.makedirs(_MEM_STATS_DIR, exist_ok=True)
165
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
166
 
167
  def _read_cgroup_memory():
168
  # cgroup v2 (modern Docker)
@@ -219,6 +255,7 @@ def get_memory_stats():
219
 
220
  stats = {
221
  "pid": os.getpid(),
 
222
  "rss_mb": rss_mb,
223
  "cache_size": cache_size,
224
  "requests": _request_count,
@@ -259,6 +296,8 @@ def start_memory_trimmer(interval_seconds=300):
259
  def _read_all_worker_files():
260
  entries = []
261
  for filename in os.listdir(_MEM_STATS_DIR):
 
 
262
  fpath = os.path.join(_MEM_STATS_DIR, filename)
263
  try:
264
  with open(fpath) as f:
@@ -284,18 +323,20 @@ def get_dashboard_data():
284
  pass
285
  continue
286
 
 
 
287
  if age > _STALE_AFTER_SECONDS:
288
- alerts.append(f"worker {stats['pid']} sem reportar há {int(age)}s")
289
  continue
290
 
291
  alive.append(stats)
292
 
293
  if (stats.get("uptime_seconds", 0) > _RSS_SPIKE_MIN_UPTIME_SECONDS
294
  and stats.get("rss_delta_2min_mb", 0) > _RSS_SPIKE_MB):
295
- alerts.append(f"worker {stats['pid']} RAM subiu {stats['rss_delta_2min_mb']:.0f} MB em 2 min")
296
 
297
  if stats.get("evictions_per_min", 0) > _CACHE_THRASH_EVICTIONS_PER_MIN:
298
- alerts.append(f"worker {stats['pid']} cache a saturar (~{stats['evictions_per_min']:.0f} evicções/min)")
299
 
300
  alive.sort(key=lambda s: s["pid"])
301
 
 
163
  _MEM_STATS_DIR = "/tmp/similarity_mem_stats"
164
  os.makedirs(_MEM_STATS_DIR, exist_ok=True)
165
 
166
+ _NAMES_FILE = os.path.join(_MEM_STATS_DIR, "_names.json")
167
+ _WORKER_NAME_ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
168
+
169
+
170
+ def _assign_worker_name():
171
+ try:
172
+ with open(_NAMES_FILE) as f:
173
+ names = json.load(f)
174
+ except (OSError, json.JSONDecodeError):
175
+ names = {}
176
+
177
+ pid_key = str(os.getpid())
178
+ if pid_key in names:
179
+ return names[pid_key]
180
+
181
+ used = set(names.values())
182
+ for letter in _WORKER_NAME_ALPHABET:
183
+ candidate = f"Worker {letter}"
184
+ if candidate not in used:
185
+ break
186
+ else:
187
+ candidate = f"Worker {len(names) + 1}"
188
+
189
+ names[pid_key] = candidate
190
+ try:
191
+ with open(_NAMES_FILE, "w") as f:
192
+ json.dump(names, f)
193
+ except OSError:
194
+ pass
195
+ return candidate
196
+
197
+
198
+ # Assigned once at import time and kept for the process's lifetime — a name is
199
+ # never reused for a different pid, so a restart is visible as a new name.
200
+ _worker_name = _assign_worker_name()
201
+
202
 
203
  def _read_cgroup_memory():
204
  # cgroup v2 (modern Docker)
 
255
 
256
  stats = {
257
  "pid": os.getpid(),
258
+ "name": _worker_name,
259
  "rss_mb": rss_mb,
260
  "cache_size": cache_size,
261
  "requests": _request_count,
 
296
  def _read_all_worker_files():
297
  entries = []
298
  for filename in os.listdir(_MEM_STATS_DIR):
299
+ if filename.startswith("_"):
300
+ continue # internal files (e.g. _names.json), not per-worker stats
301
  fpath = os.path.join(_MEM_STATS_DIR, filename)
302
  try:
303
  with open(fpath) as f:
 
323
  pass
324
  continue
325
 
326
+ name = stats.get("name") or f"pid {stats['pid']}"
327
+
328
  if age > _STALE_AFTER_SECONDS:
329
+ alerts.append(f"{name} sem reportar há {int(age)}s")
330
  continue
331
 
332
  alive.append(stats)
333
 
334
  if (stats.get("uptime_seconds", 0) > _RSS_SPIKE_MIN_UPTIME_SECONDS
335
  and stats.get("rss_delta_2min_mb", 0) > _RSS_SPIKE_MB):
336
+ alerts.append(f"{name} RAM subiu {stats['rss_delta_2min_mb']:.0f} MB em 2 min")
337
 
338
  if stats.get("evictions_per_min", 0) > _CACHE_THRASH_EVICTIONS_PER_MIN:
339
+ alerts.append(f"{name} cache a saturar (~{stats['evictions_per_min']:.0f} evicções/min)")
340
 
341
  alive.sort(key=lambda s: s["pid"])
342