app/inference.py

"""Hugging Face LLM helper for AutoApp Builder.

Generates real application plans and source files through the free serverless
Inference Providers, with model fallback.
"""
from __future__ import annotations

import os
from typing import Optional

from huggingface_hub import InferenceClient

_TOKEN_ENV_KEYS = (
    "HF_TOKEN",
    "HUGGING_FACE_HUB_TOKEN",
    "HUGGINGFACEHUB_API_TOKEN",
    "HF_API_TOKEN",
)

TEXT_MODELS = [
    m
    for m in (
        os.environ.get("TEXT_MODEL"),
        "Qwen/Qwen2.5-Coder-32B-Instruct",
        "Qwen/Qwen2.5-7B-Instruct",
        "meta-llama/Llama-3.1-8B-Instruct",
        "mistralai/Mistral-7B-Instruct-v0.3",
    )
    if m
]


class InferenceError(RuntimeError):
    """Raised when no model could satisfy a request."""


def get_token() -> Optional[str]:
    for key in _TOKEN_ENV_KEYS:
        value = os.environ.get(key)
        if value and value.strip():
            return value.strip()
    return None


def get_client(timeout: int = 120) -> InferenceClient:
    token = get_token()
    if not token:
        raise InferenceError(
            "No Hugging Face token configured. Set HF_TOKEN (with the 'Make calls "
            "to Inference Providers' permission) as a Space secret."
        )
    return InferenceClient(token=token, timeout=timeout)


def chat(messages: list[dict], max_tokens: int = 2200, temperature: float = 0.4) -> str:
    client = get_client()
    errors: list[str] = []
    for model in TEXT_MODELS:
        try:
            out = client.chat_completion(
                model=model,
                messages=messages,
                max_tokens=max_tokens,
                temperature=temperature,
            )
            return out.choices[0].message.content or ""
        except Exception as exc:  # noqa: BLE001
            errors.append(f"{model}: {type(exc).__name__}: {str(exc)[:120]}")
            if "403" in str(exc):
                raise InferenceError(
                    "Inference refused (403). The HF_TOKEN needs the 'Make calls to "
                    "Inference Providers' permission.\n" + "\n".join(errors[-3:])
                ) from exc
    raise InferenceError("All text models failed:\n" + "\n".join(errors[-4:]))