Download backend/flagship/worker.py from Gaurav711/HyperFlow: direct link, hf CLI and curl.
- Browser
- Download file 4.6 kB
-
https://huggingface.co/spaces/Gaurav711/HyperFlow/resolve/main/backend/flagship/worker.py
- Command line
-
hf download hf://spaces/Gaurav711/HyperFlow/backend/flagship/worker.py
-
curl -L -o worker.py https://huggingface.co/spaces/Gaurav711/HyperFlow/resolve/main/backend/flagship/worker.py
4.6 kB
| """Durable job worker and optional outbox notifications; PostgreSQL remains authoritative.""" | |
| import argparse | |
| import json | |
| import os | |
| import time | |
| import threading | |
| try: | |
| from dotenv import load_dotenv | |
| load_dotenv() | |
| except ImportError: | |
| pass | |
| from sqlalchemy import select, update | |
| from .store import Store, events | |
| from .identity import Identity | |
| PRUNE_INTERVAL_SECONDS = 600 | |
| def poll_delay(did_work): | |
| """Seconds to wait before the next tick. Idle polling can be slowed so a serverless database | |
| scales to zero between polls; queued work is still drained every 2 seconds.""" | |
| return 2 if did_work else max(2, int(os.getenv('HYPERFLOW_WORKER_IDLE_SECONDS', '2'))) | |
| def stale_after_seconds(): | |
| """Heartbeat age after which the worker counts as down: three idle intervals, at least 90 s.""" | |
| return max(90, 3 * poll_delay(did_work=False)) | |
| def heartbeat(store): | |
| try: | |
| beat = store.get('worker', 'worker-heartbeat', '__system__') | |
| # No event: a heartbeat every few seconds would grow the event log without bound. | |
| store.update('worker', beat['id'], '__system__', {'heartbeat': time.time()}, beat['version'], event=False) | |
| except LookupError: | |
| store.create('worker', '__system__', {'heartbeat': time.time()}, 'worker-heartbeat') | |
| def tick(store): | |
| from .ml import train_job | |
| worked = False | |
| with store.atomic('__system__', 'worker'): | |
| heartbeat(store) | |
| for job in store.jobs(): | |
| if job['data']['state'] not in ('QUEUED', 'RUNNING'): | |
| continue | |
| data = job['data'] | |
| attempts = data.get('attempts', 0) + 1 | |
| if attempts > 3: | |
| store.update('training_job', job['id'], job['account_id'], {**data, 'state': 'FAILED', 'reason': 'Worker retry budget exhausted'}, job['version']) | |
| continue | |
| job = store.update('training_job', job['id'], job['account_id'], {**data, 'state': 'RUNNING', 'attempts': attempts}, job['version']) | |
| stop = threading.Event() | |
| def keep_alive(): | |
| while not stop.wait(10): | |
| try: | |
| heartbeat(store) | |
| except Exception: | |
| # Readiness must become unhealthy if persistence is lost. | |
| break | |
| monitor = threading.Thread(target=keep_alive, daemon=True) | |
| monitor.start() | |
| try: | |
| result = train_job({**data.get('payload', {}), 'auto_promote': False}) | |
| terminal = result.get('status', 'FAILED') | |
| if terminal not in {'EVALUATED', 'INSUFFICIENT_DATA', 'REJECTED', 'FAILED'}: | |
| terminal = 'FAILED' | |
| store.update('training_job', job['id'], job['account_id'], {**job['data'], 'state': terminal, 'result': result}, job['version']) | |
| except Exception as exc: | |
| store.update('training_job', job['id'], job['account_id'], {**job['data'], 'state': 'FAILED', 'reason': type(exc).__name__}, job['version']) | |
| finally: | |
| stop.set() | |
| monitor.join(timeout=2) | |
| worked = True | |
| break | |
| # Redis is only a wakeup path. Cursor-based SSE reads durable rows directly. | |
| if os.getenv('REDIS_URL'): | |
| try: | |
| import redis | |
| client = redis.Redis.from_url(os.environ['REDIS_URL'], socket_timeout=1) | |
| with store.engine.connect() as c: | |
| pending = c.execute(select(events).where(events.c.notified == 0).order_by(events.c.id).limit(100)).all() | |
| for row in pending: | |
| client.publish('hyperflow:events', json.dumps({'event_id': row.id, 'account_id': row.account_id})) | |
| with store.engine.begin() as c: | |
| c.execute(update(events).where(events.c.id == row.id).values(notified=1)) | |
| except Exception: | |
| pass | |
| return worked | |
| def main(): | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument('--once', action='store_true') | |
| args = parser.parse_args() | |
| store = Store() | |
| if not store.ready(): | |
| raise SystemExit('Run python -m backend.flagship.manage migrate first') | |
| last_prune = 0.0 | |
| while True: | |
| worked = tick(store) | |
| if time.monotonic() - last_prune >= PRUNE_INTERVAL_SECONDS or args.once: | |
| Identity(store).prune() | |
| last_prune = time.monotonic() | |
| if args.once: | |
| break | |
| time.sleep(poll_delay(worked)) | |
| if __name__ == '__main__': | |
| main() | |