HyperFlow / backend /flagship /worker.py
Gaurav711's picture
Deploy 35b1393
64b0986 verified
Raw History Blame Contribute Delete
4.6 kB
"""Durable job worker and optional outbox notifications; PostgreSQL remains authoritative."""
import argparse
import json
import os
import time
import threading
try:
from dotenv import load_dotenv
load_dotenv()
except ImportError:
pass
from sqlalchemy import select, update
from .store import Store, events
from .identity import Identity
PRUNE_INTERVAL_SECONDS = 600
def poll_delay(did_work):
"""Seconds to wait before the next tick. Idle polling can be slowed so a serverless database
scales to zero between polls; queued work is still drained every 2 seconds."""
return 2 if did_work else max(2, int(os.getenv('HYPERFLOW_WORKER_IDLE_SECONDS', '2')))
def stale_after_seconds():
"""Heartbeat age after which the worker counts as down: three idle intervals, at least 90 s."""
return max(90, 3 * poll_delay(did_work=False))
def heartbeat(store):
try:
beat = store.get('worker', 'worker-heartbeat', '__system__')
# No event: a heartbeat every few seconds would grow the event log without bound.
store.update('worker', beat['id'], '__system__', {'heartbeat': time.time()}, beat['version'], event=False)
except LookupError:
store.create('worker', '__system__', {'heartbeat': time.time()}, 'worker-heartbeat')
def tick(store):
from .ml import train_job
worked = False
with store.atomic('__system__', 'worker'):
heartbeat(store)
for job in store.jobs():
if job['data']['state'] not in ('QUEUED', 'RUNNING'):
continue
data = job['data']
attempts = data.get('attempts', 0) + 1
if attempts > 3:
store.update('training_job', job['id'], job['account_id'], {**data, 'state': 'FAILED', 'reason': 'Worker retry budget exhausted'}, job['version'])
continue
job = store.update('training_job', job['id'], job['account_id'], {**data, 'state': 'RUNNING', 'attempts': attempts}, job['version'])
stop = threading.Event()
def keep_alive():
while not stop.wait(10):
try:
heartbeat(store)
except Exception:
# Readiness must become unhealthy if persistence is lost.
break
monitor = threading.Thread(target=keep_alive, daemon=True)
monitor.start()
try:
result = train_job({**data.get('payload', {}), 'auto_promote': False})
terminal = result.get('status', 'FAILED')
if terminal not in {'EVALUATED', 'INSUFFICIENT_DATA', 'REJECTED', 'FAILED'}:
terminal = 'FAILED'
store.update('training_job', job['id'], job['account_id'], {**job['data'], 'state': terminal, 'result': result}, job['version'])
except Exception as exc:
store.update('training_job', job['id'], job['account_id'], {**job['data'], 'state': 'FAILED', 'reason': type(exc).__name__}, job['version'])
finally:
stop.set()
monitor.join(timeout=2)
worked = True
break
# Redis is only a wakeup path. Cursor-based SSE reads durable rows directly.
if os.getenv('REDIS_URL'):
try:
import redis
client = redis.Redis.from_url(os.environ['REDIS_URL'], socket_timeout=1)
with store.engine.connect() as c:
pending = c.execute(select(events).where(events.c.notified == 0).order_by(events.c.id).limit(100)).all()
for row in pending:
client.publish('hyperflow:events', json.dumps({'event_id': row.id, 'account_id': row.account_id}))
with store.engine.begin() as c:
c.execute(update(events).where(events.c.id == row.id).values(notified=1))
except Exception:
pass
return worked
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--once', action='store_true')
args = parser.parse_args()
store = Store()
if not store.ready():
raise SystemExit('Run python -m backend.flagship.manage migrate first')
last_prune = 0.0
while True:
worked = tick(store)
if time.monotonic() - last_prune >= PRUNE_INTERVAL_SECONDS or args.once:
Identity(store).prune()
last_prune = time.monotonic()
if args.once:
break
time.sleep(poll_delay(worked))
if __name__ == '__main__':
main()