ghostdrive1 commited on
Commit
3e62ed4
Β·
1 Parent(s): 190b4da

v26: wire SpacePromoter + Discord bot into lifespan

Browse files
Files changed (1) hide show
  1. packages/brain/main.py +75 -6
packages/brain/main.py CHANGED
@@ -9,10 +9,12 @@ Startup sequence:
9
  3. TaskDispatcher instantiated with pool
10
  4. Memory pipeline: Embedder + ZillizStore + RaptorTree + MemoryWorker
11
  5. LifecycleEngine + GroundTruthStore + RDLoop (if Redis available)
 
12
  6. Sentinel instantiated (if GEMINI_SENTINEL_KEY set)
13
  7. Council instantiated (always β€” uses general pool)
14
  8. Background tasks: health-ping + memory flush worker
15
  9. FastAPI app begins serving on port 7860 (HF Spaces standard)
 
16
 
17
  Endpoints:
18
  POST /infer β€” Discord bot -> Brain. Auth: X-Ultron-Token header.
@@ -29,6 +31,8 @@ Design decisions:
29
  - /health is intentionally unauthenticated β€” CF Worker + Sentinel ping without token.
30
  - Structured JSON responses everywhere. Discord-formatted strings only at bot layer.
31
  - Request IDs injected via middleware for distributed tracing readiness.
 
 
32
 
33
  Future bug risks (pre-registered):
34
  M1 [HIGH] HF Spaces can spin up MULTIPLE workers for the same Space on scale events.
@@ -80,12 +84,22 @@ Future bug risks (pre-registered):
80
  triggered externally (post-task completion). If called from /infer handler,
81
  it blocks the response. Fix: always asyncio.create_task() for RDLoop.run().
82
 
83
- Tool calls used writing this file (v25):
84
- Github:get_file_contents x1 (lifecycle.py interface)
85
- Github:get_file_contents x1 (ground_truth.py interface)
86
- Github:get_file_contents x1 (rd_loop.py interface)
 
 
 
 
 
 
 
 
 
87
  Github:get_file_contents x1 (main.py current state + sha)
88
- pipecat-ai/pipecat: src/pipecat/services/groq/stt.py (Whisper API pattern)
 
89
  """
90
 
91
  from __future__ import annotations
@@ -95,6 +109,7 @@ import hmac
95
  import json
96
  import logging
97
  import os
 
98
  import time
99
  import uuid
100
  from contextlib import asynccontextmanager
@@ -112,6 +127,8 @@ from packages.brain.task_dispatcher import TaskDispatcher
112
  from packages.brain.llm_router import make_provider_llm_fn
113
  from packages.shared.config import get_settings
114
  from packages.shared.exceptions import AllKeysExhaustedError, SentinelKeyUnavailableError
 
 
115
 
116
  logger = logging.getLogger(__name__)
117
  logging.basicConfig(
@@ -287,6 +304,17 @@ async def lifespan(app: FastAPI):
287
  else:
288
  logger.warning("[Startup] Lifecycle/GT/RDLoop DISABLED β€” Redis not available")
289
 
 
 
 
 
 
 
 
 
 
 
 
290
  # ── Step 6: Sentinel (optional β€” degrades gracefully if key unset) ────
291
  sentinel = None
292
  try:
@@ -323,13 +351,37 @@ async def lifespan(app: FastAPI):
323
  _health_ping_loop(_brain_url, interval_seconds=43200)
324
  )
325
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
326
  elapsed = (time.monotonic() - startup_start) * 1000
327
  lifecycle_active = hasattr(app.state, "lifecycle") and app.state.lifecycle is not None
328
  logger.info(
329
  f"[Startup] Ultron V4 Brain READY in {elapsed:.1f}ms. "
330
  f"Pool general={len(pool.general)} sentinel={'ACTIVE' if sentinel else 'INACTIVE'} "
331
  f"council=ACTIVE memory={'ACTIVE' if memory_worker_task else 'INACTIVE'} "
332
- f"lifecycle={'ACTIVE' if lifecycle_active else 'INACTIVE'}"
 
 
333
  )
334
 
335
  # ── Yield: serve requests ─────────────────────────────────────────────
@@ -337,6 +389,7 @@ async def lifespan(app: FastAPI):
337
 
338
  # ── Shutdown ──────────────────────────────────────────────────────────
339
  logger.info("[Shutdown] Cancelling background tasks...")
 
340
  _ping_task.cancel()
341
  if memory_worker_task:
342
  memory_worker_task.cancel()
@@ -344,6 +397,12 @@ async def lifespan(app: FastAPI):
344
  await memory_worker_task
345
  except asyncio.CancelledError:
346
  pass
 
 
 
 
 
 
347
  try:
348
  await _ping_task
349
  except asyncio.CancelledError:
@@ -352,6 +411,7 @@ async def lifespan(app: FastAPI):
352
  await redis_client.aclose()
353
  if hasattr(app.state, "zilliz_store"):
354
  await app.state.zilliz_store.close()
 
355
  logger.info("[Shutdown] Ultron V4 Brain stopped cleanly.")
356
 
357
 
@@ -398,6 +458,13 @@ async def health(request: Request) -> JSONResponse:
398
  uptime = time.monotonic() - request.app.state.start_time
399
  status = "ok" if pool_status["general_available"] > 0 else "degraded"
400
 
 
 
 
 
 
 
 
401
  return JSONResponse({
402
  "status": status,
403
  "uptime_seconds": round(uptime, 1),
@@ -406,6 +473,8 @@ async def health(request: Request) -> JSONResponse:
406
  "sentinel_active": request.app.state.sentinel is not None,
407
  "council_active": hasattr(request.app.state, "council"),
408
  "lifecycle_active": hasattr(request.app.state, "lifecycle") and request.app.state.lifecycle is not None,
 
 
409
  "pool": {
410
  "general_available": pool_status["general_available"],
411
  "general_total": len(pool_status["general"]),
 
9
  3. TaskDispatcher instantiated with pool
10
  4. Memory pipeline: Embedder + ZillizStore + RaptorTree + MemoryWorker
11
  5. LifecycleEngine + GroundTruthStore + RDLoop (if Redis available)
12
+ 5c. SpacePromoter (optional β€” health-check loop + CF KV promotion)
13
  6. Sentinel instantiated (if GEMINI_SENTINEL_KEY set)
14
  7. Council instantiated (always β€” uses general pool)
15
  8. Background tasks: health-ping + memory flush worker
16
  9. FastAPI app begins serving on port 7860 (HF Spaces standard)
17
+ 10. Discord bot: blocking .run() in daemon thread (lifecycle-aware)
18
 
19
  Endpoints:
20
  POST /infer β€” Discord bot -> Brain. Auth: X-Ultron-Token header.
 
31
  - /health is intentionally unauthenticated β€” CF Worker + Sentinel ping without token.
32
  - Structured JSON responses everywhere. Discord-formatted strings only at bot layer.
33
  - Request IDs injected via middleware for distributed tracing readiness.
34
+ - Discord bot runs in daemon thread (discord.py owns its own asyncio event loop).
35
+ - SpacePromoter runs as asyncio task inside FastAPI's event loop (pure async).
36
 
37
  Future bug risks (pre-registered):
38
  M1 [HIGH] HF Spaces can spin up MULTIPLE workers for the same Space on scale events.
 
84
  triggered externally (post-task completion). If called from /infer handler,
85
  it blocks the response. Fix: always asyncio.create_task() for RDLoop.run().
86
 
87
+ M7 [MED] Discord bot thread holds a reference to redis_client (aioredis). aioredis
88
+ client is created in FastAPI's asyncio event loop. Bot thread runs its own
89
+ event loop (discord.py). Cross-loop Redis calls from bot thread will raise
90
+ "bound to different event loop". Fix v26: bot thread calls Brain /infer HTTP
91
+ (already the architecture) β€” never calls Redis directly from bot thread.
92
+ No issue in current design; flag for future if bot ever goes direct-Redis.
93
+
94
+ M8 [LOW] SpacePromoter _promoter_stop event created in lifespan scope; if lifespan
95
+ exits before promoter task starts (edge case on fast shutdown), stop event
96
+ is set before task reads it β€” task exits immediately. Acceptable: only on
97
+ startup crash scenarios.
98
+
99
+ Tool calls used writing this file (v26):
100
  Github:get_file_contents x1 (main.py current state + sha)
101
+ Github:get_file_contents x1 (discord_bot.py β€” run() signature)
102
+ Github:get_file_contents x1 (space_promoter.py β€” SpacePromoter.run() signature)
103
  """
104
 
105
  from __future__ import annotations
 
109
  import json
110
  import logging
111
  import os
112
+ import threading
113
  import time
114
  import uuid
115
  from contextlib import asynccontextmanager
 
127
  from packages.brain.llm_router import make_provider_llm_fn
128
  from packages.shared.config import get_settings
129
  from packages.shared.exceptions import AllKeysExhaustedError, SentinelKeyUnavailableError
130
+ from packages.brain import discord_bot as _discord_bot # type: ignore
131
+ from packages.infrastructure.space_promoter import SpacePromoter
132
 
133
  logger = logging.getLogger(__name__)
134
  logging.basicConfig(
 
304
  else:
305
  logger.warning("[Startup] Lifecycle/GT/RDLoop DISABLED β€” Redis not available")
306
 
307
+ # ── Step 5c: SpacePromoter (optional β€” health-check + CF KV promotion) ─
308
+ _promoter_stop: asyncio.Event = asyncio.Event()
309
+ _promoter_task: Optional[asyncio.Task] = None
310
+ try:
311
+ promoter = SpacePromoter(redis_client=redis_client)
312
+ _promoter_task = asyncio.create_task(promoter.run(_promoter_stop))
313
+ app.state.promoter = promoter
314
+ logger.info("[Startup] SpacePromoter: ACTIVE")
315
+ except Exception as e:
316
+ logger.warning(f"[Startup] SpacePromoter init failed (non-fatal): {e}")
317
+
318
  # ── Step 6: Sentinel (optional β€” degrades gracefully if key unset) ────
319
  sentinel = None
320
  try:
 
351
  _health_ping_loop(_brain_url, interval_seconds=43200)
352
  )
353
 
354
+ # ── Step 10: Discord bot (daemon thread β€” discord.py owns its own event loop) ──
355
+ # M7: bot only calls Brain /infer via HTTP, never touches Redis directly β€”
356
+ # no cross-loop aioredis issue. Passing redis_client as reference is safe
357
+ # because bot module receives it but only the on_message handler uses it
358
+ # for Redis calls that run inside its own thread's event loop via discord.py.
359
+ # NOTE: discord_bot._ctx_append/_ctx_get use asyncio internally. They run
360
+ # in the bot's own event loop (created by discord.py in the bot thread) β€”
361
+ # NOT in FastAPI's loop. The aioredis client created here is bound to
362
+ # FastAPI's loop. WORKAROUND: bot module must create its OWN aioredis client
363
+ # internally if Redis calls needed. For now, Redis is passed but aioredis
364
+ # may raise cross-loop errors β€” tracked as M7. Mitigation: pass redis=None
365
+ # until bot-side Redis is refactored to create its own client.
366
+ _bot_lifecycle = getattr(app.state, "lifecycle", None)
367
+ _bot_thread = threading.Thread(
368
+ target=_discord_bot.run,
369
+ kwargs={"redis": None, "lifecycle": _bot_lifecycle}, # M7: redis=None until bot refactor
370
+ daemon=True,
371
+ name="ultron-discord-bot",
372
+ )
373
+ _bot_thread.start()
374
+ logger.info("[Startup] Discord bot thread started.")
375
+
376
  elapsed = (time.monotonic() - startup_start) * 1000
377
  lifecycle_active = hasattr(app.state, "lifecycle") and app.state.lifecycle is not None
378
  logger.info(
379
  f"[Startup] Ultron V4 Brain READY in {elapsed:.1f}ms. "
380
  f"Pool general={len(pool.general)} sentinel={'ACTIVE' if sentinel else 'INACTIVE'} "
381
  f"council=ACTIVE memory={'ACTIVE' if memory_worker_task else 'INACTIVE'} "
382
+ f"lifecycle={'ACTIVE' if lifecycle_active else 'INACTIVE'} "
383
+ f"promoter={'ACTIVE' if _promoter_task else 'INACTIVE'} "
384
+ f"discord_bot=ACTIVE"
385
  )
386
 
387
  # ── Yield: serve requests ─────────────────────────────────────────────
 
389
 
390
  # ── Shutdown ──────────────────────────────────────────────────────────
391
  logger.info("[Shutdown] Cancelling background tasks...")
392
+ _promoter_stop.set() # signal SpacePromoter to exit cleanly
393
  _ping_task.cancel()
394
  if memory_worker_task:
395
  memory_worker_task.cancel()
 
397
  await memory_worker_task
398
  except asyncio.CancelledError:
399
  pass
400
+ if _promoter_task:
401
+ _promoter_task.cancel()
402
+ try:
403
+ await _promoter_task
404
+ except asyncio.CancelledError:
405
+ pass
406
  try:
407
  await _ping_task
408
  except asyncio.CancelledError:
 
411
  await redis_client.aclose()
412
  if hasattr(app.state, "zilliz_store"):
413
  await app.state.zilliz_store.close()
414
+ # Discord bot thread is daemon β€” dies with process. No explicit join needed.
415
  logger.info("[Shutdown] Ultron V4 Brain stopped cleanly.")
416
 
417
 
 
458
  uptime = time.monotonic() - request.app.state.start_time
459
  status = "ok" if pool_status["general_available"] > 0 else "degraded"
460
 
461
+ promoter_status = None
462
+ if hasattr(request.app.state, "promoter"):
463
+ try:
464
+ promoter_status = request.app.state.promoter.get_status()
465
+ except Exception:
466
+ pass
467
+
468
  return JSONResponse({
469
  "status": status,
470
  "uptime_seconds": round(uptime, 1),
 
473
  "sentinel_active": request.app.state.sentinel is not None,
474
  "council_active": hasattr(request.app.state, "council"),
475
  "lifecycle_active": hasattr(request.app.state, "lifecycle") and request.app.state.lifecycle is not None,
476
+ "promoter_active": hasattr(request.app.state, "promoter"),
477
+ "promoter": promoter_status,
478
  "pool": {
479
  "general_available": pool_status["general_available"],
480
  "general_total": len(pool_status["general"]),