app.py

import time
import threading
from huggingface_hub import hf_hub_download
from llama_cpp.server.app import create_app
from llama_cpp.server.settings import ModelSettings, ServerSettings

# Download the model
model_path = hf_hub_download(
    repo_id="amkkk/Qwen3.5-0.8B-GGUF-uncensored-opus-distill",
    filename="qwen3.5-0.8b-ablate-e2-opus46-postopus-runtime.Q4_K_M.gguf",
    repo_type="model"
)

# Configure model settings
model_settings = ModelSettings(
    model=model_path,
    n_ctx=8192,
    n_threads=4,
    n_gpu_layers=0,
    chat_format="qwen",
    verbose=False
)

# Create server settings
server_settings = ServerSettings(
    host="0.0.0.0",
    port=7860
)

# Create the FastAPI app
app = create_app(
    model_settings=[model_settings],
    server_settings=server_settings
)

print("✅ Server starting...")

if __name__ == "__main__":
    import uvicorn
    uvicorn.run(app, host="0.0.0.0", port=7860)