"""Durable job worker and optional outbox notifications; PostgreSQL remains authoritative.""" import argparse import json import os import time import threading try: from dotenv import load_dotenv load_dotenv() except ImportError: pass from sqlalchemy import select, update from .store import Store, events from .identity import Identity PRUNE_INTERVAL_SECONDS = 600 def poll_delay(did_work): """Seconds to wait before the next tick. Idle polling can be slowed so a serverless database scales to zero between polls; queued work is still drained every 2 seconds.""" return 2 if did_work else max(2, int(os.getenv('HYPERFLOW_WORKER_IDLE_SECONDS', '2'))) def stale_after_seconds(): """Heartbeat age after which the worker counts as down: three idle intervals, at least 90 s.""" return max(90, 3 * poll_delay(did_work=False)) def heartbeat(store): try: beat = store.get('worker', 'worker-heartbeat', '__system__') # No event: a heartbeat every few seconds would grow the event log without bound. store.update('worker', beat['id'], '__system__', {'heartbeat': time.time()}, beat['version'], event=False) except LookupError: store.create('worker', '__system__', {'heartbeat': time.time()}, 'worker-heartbeat') def tick(store): from .ml import train_job worked = False with store.atomic('__system__', 'worker'): heartbeat(store) for job in store.jobs(): if job['data']['state'] not in ('QUEUED', 'RUNNING'): continue data = job['data'] attempts = data.get('attempts', 0) + 1 if attempts > 3: store.update('training_job', job['id'], job['account_id'], {**data, 'state': 'FAILED', 'reason': 'Worker retry budget exhausted'}, job['version']) continue job = store.update('training_job', job['id'], job['account_id'], {**data, 'state': 'RUNNING', 'attempts': attempts}, job['version']) stop = threading.Event() def keep_alive(): while not stop.wait(10): try: heartbeat(store) except Exception: # Readiness must become unhealthy if persistence is lost. break monitor = threading.Thread(target=keep_alive, daemon=True) monitor.start() try: result = train_job({**data.get('payload', {}), 'auto_promote': False}) terminal = result.get('status', 'FAILED') if terminal not in {'EVALUATED', 'INSUFFICIENT_DATA', 'REJECTED', 'FAILED'}: terminal = 'FAILED' store.update('training_job', job['id'], job['account_id'], {**job['data'], 'state': terminal, 'result': result}, job['version']) except Exception as exc: store.update('training_job', job['id'], job['account_id'], {**job['data'], 'state': 'FAILED', 'reason': type(exc).__name__}, job['version']) finally: stop.set() monitor.join(timeout=2) worked = True break # Redis is only a wakeup path. Cursor-based SSE reads durable rows directly. if os.getenv('REDIS_URL'): try: import redis client = redis.Redis.from_url(os.environ['REDIS_URL'], socket_timeout=1) with store.engine.connect() as c: pending = c.execute(select(events).where(events.c.notified == 0).order_by(events.c.id).limit(100)).all() for row in pending: client.publish('hyperflow:events', json.dumps({'event_id': row.id, 'account_id': row.account_id})) with store.engine.begin() as c: c.execute(update(events).where(events.c.id == row.id).values(notified=1)) except Exception: pass return worked def main(): parser = argparse.ArgumentParser() parser.add_argument('--once', action='store_true') args = parser.parse_args() store = Store() if not store.ready(): raise SystemExit('Run python -m backend.flagship.manage migrate first') last_prune = 0.0 while True: worked = tick(store) if time.monotonic() - last_prune >= PRUNE_INTERVAL_SECONDS or args.once: Identity(store).prune() last_prune = time.monotonic() if args.once: break time.sleep(poll_delay(worked)) if __name__ == '__main__': main()