#!/usr/bin/env python3 """WHOAMISec Wizard-Vicuna-13B — OpenAI-compatible API via Ollama (zero compilation).""" import os, sys, json, time, subprocess from starlette.applications import Starlette from starlette.routing import Route from starlette.responses import JSONResponse, StreamingResponse from starlette.middleware.cors import CORSMiddleware import httpx MODEL_DIR = "/data/model" MODEL_FILE = os.path.join(MODEL_DIR, "Wizard-Vicuna-13B-Uncensored.Q4_K_M.gguf") MODEL_REPO = "TheBloke/Wizard-Vicuna-13B-Uncensored-GGUF" MODEL_FILENAME = "Wizard-Vicuna-13B-Uncensored.Q4_K_M.gguf" MODEL_NAME = "whoamisec-wizard-13b" OLLAMA_URL = "http://localhost:11434" ready = False def download_model(): os.makedirs(MODEL_DIR, exist_ok=True) if os.path.exists(MODEL_FILE): print(f"[SRV] Model cached: {MODEL_FILE}", flush=True) return print(f"[SRV] Downloading {MODEL_FILENAME}...", flush=True) hf_token = os.environ.get("HF_TOKEN", "") from huggingface_hub import hf_hub_download path = hf_hub_download(MODEL_REPO, MODEL_FILENAME, local_dir=MODEL_DIR, local_dir_use_symlinks=False, token=hf_token or None) print(f"[SRV] Downloaded: {path}", flush=True) def start_ollama(): global ready # Step 1: Download model FIRST download_model() # Step 2: Start Ollama server print("[SRV] Starting Ollama server...", flush=True) subprocess.Popen( ["ollama", "serve"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL ) for _ in range(30): try: r = httpx.get(f"{OLLAMA_URL}/api/tags", timeout=2) if r.status_code == 200: print("[SRV] Ollama ready!", flush=True) break except: pass time.sleep(1) # Step 3: Generate Modelfile dynamically modelfile_content = f'''FROM {MODEL_FILE} TEMPLATE """{{{{ if .System }}}}{{{{ .System }}}} {{{{ end }}}}{{{{ range .Messages }}}}{{{{ if eq .Role "user" }}}} USER: {{{{ .Content }}}} ASSISTANT: {{{{ end }}}} {{{{ end }}}} """ PARAMETER stop "" PARAMETER num_ctx 2048 ''' mf_path = "/tmp/Modelfile" with open(mf_path, "w") as f: f.write(modelfile_content) # Step 4: Create model in Ollama print(f"[SRV] Creating model '{MODEL_NAME}' from {MODEL_FILE}...", flush=True) result = subprocess.run( ["ollama", "create", MODEL_NAME, "-f", mf_path], capture_output=True, text=True ) if result.returncode != 0: print(f"[SRV] ERROR creating model: {result.stderr[:500]}", flush=True) sys.exit(1) print(f"[SRV] Model '{MODEL_NAME}' ready!", flush=True) ready = True # Start everything in background (download + ollama setup) import threading threading.Thread(target=start_ollama, daemon=True).start() async def chat_completions(request): if not ready: return JSONResponse({"error": {"message": "model loading", "type": "server_error"}}, status_code=503) try: body = await request.json() async with httpx.AsyncClient(timeout=600) as client: if body.get("stream", False): resp = await client.post( f"{OLLAMA_URL}/v1/chat/completions", json={**body, "model": MODEL_NAME}, headers={"Content-Type": "application/json"}, ) return StreamingResponse( resp.aiter_bytes(), media_type=resp.headers.get("content-type", "text/event-stream"), headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, ) else: resp = await client.post( f"{OLLAMA_URL}/v1/chat/completions", json={**body, "model": MODEL_NAME}, headers={"Content-Type": "application/json"}, ) data = resp.json() if "model" in data: data["model"] = body.get("model", MODEL_NAME) return JSONResponse(data) except Exception as e: return JSONResponse({"error": {"message": str(e), "type": "server_error"}}, status_code=500) async def models(request): return JSONResponse({ "object": "list", "data": [{"id": MODEL_NAME, "object": "model", "owned_by": "whoamisec", "permissions": [{"allow_sampling": True, "allow_logprobs": True, "allow_search_indices": True, "allow_view": True, "allow_fine_tuning": False}]}], }) async def health(request): if ready: try: async with httpx.AsyncClient(timeout=5) as client: r = await client.get(f"{OLLAMA_URL}/api/tags") if r.status_code == 200: return JSONResponse({"status": "ok", "model": MODEL_NAME}) except: pass return JSONResponse({"status": "loading", "model": MODEL_NAME}) app = Starlette(debug=False, routes=[ Route("/v1/chat/completions", chat_completions, methods=["POST"]), Route("/v1/models", models, methods=["GET"]), Route("/health", health, methods=["GET"]), ]) app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"]) if __name__ == "__main__": import uvicorn uvicorn.run(app, host="0.0.0.0", port=7860, log_level="warning")