start_server.py

import os
import subprocess
import sys
from pathlib import Path

from huggingface_hub import hf_hub_download


def read_env(name: str, default: str) -> str:
    value = os.getenv(name, default).strip()
    return value or default


def main() -> None:
    repo_id = read_env("MODEL_REPO", "unsloth/Qwen3.5-0.8B-GGUF")
    filename = read_env("MODEL_FILE", "Qwen3.5-0.8B-Q4_K_M.gguf")
    model_dir = Path(read_env("MODEL_DIR", "/tmp/models"))
    port = read_env("PORT", "7860")
    n_ctx = read_env("N_CTX", "4096")
    n_threads = read_env("N_THREADS", "4")
    chat_format = read_env("CHAT_FORMAT", "chatml")
    api_key = os.getenv("API_KEY", "").strip()

    model_dir.mkdir(parents=True, exist_ok=True)

    token = os.getenv("HF_TOKEN", "").strip() or os.getenv("HUGGING_FACE_HUB_TOKEN", "").strip() or None
    model_path = hf_hub_download(
        repo_id=repo_id,
        filename=filename,
        token=token,
        local_dir=str(model_dir),
    )

    command = [
        sys.executable,
        "-m",
        "llama_cpp.server",
        "--model",
        model_path,
        "--host",
        "0.0.0.0",
        "--port",
        port,
        "--n_ctx",
        n_ctx,
        "--n_threads",
        n_threads,
        "--chat_format",
        chat_format,
    ]

    if api_key:
        command.extend(["--api_key", api_key])

    print("Starting OpenAI-compatible model server:")
    print(" ".join(command))
    subprocess.run(command, check=True)


if __name__ == "__main__":
    main()