core/modal_backend.py

"""Space-side client for the Modal 8B text service.

Used by core.inference.text_generate when BB_INFERENCE=modal. The Modal app
(modal_app.py, deployed separately) runs MiniCPM4.1-8B in its own transformers
env on a warm GPU. Auth comes from MODAL_TOKEN_ID / MODAL_TOKEN_SECRET in the
environment (set as Space secrets).
"""

from __future__ import annotations

MODAL_APP = "budgetbuddy-8b"
MODAL_CLASS = "Engine"

_engine = None


def _engine_handle():
    global _engine
    if _engine is None:
        import modal
        Engine = modal.Cls.from_name(MODAL_APP, MODAL_CLASS)
        _engine = Engine()
    return _engine


def text_generate(messages, max_new_tokens: int = 512, enable_thinking: bool = False) -> str:
    return _engine_handle().generate.remote(
        messages=messages, max_new_tokens=max_new_tokens, enable_thinking=enable_thinking
    )