workers/inference_worker.py

import asyncio

from queues.tasks import inference_queue
from engines.chat.manager import ChatManager


manager = ChatManager()


async def process_jobs():
    while True:
        job = await inference_queue.get()

        result = await manager.generate(
            provider=job["provider"],
            model=job["model"],
            prompt=job["prompt"],
        )

        print(result)

        inference_queue.task_done()


if __name__ == "__main__":
    asyncio.run(process_jobs())