ok

Dockerfile

@@ -0,0 +1,46 @@
+# Use Python 3.11 Slim as requested
+FROM python:3.11-slim
+
+# Set environment variables
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    # Hugging Face Spaces specific port
+    PORT=7860 \
+    # Configure cache to be writable by non-root user
+    HF_HOME=/app/cache \
+    TRANSFORMERS_CACHE=/app/cache
+
+WORKDIR /app
+
+# 1. Install system tools (needed for compiling llama-cpp if wheels miss)
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    curl \
+    && rm -rf /var/lib/apt/lists/*
+
+# 2. Create a non-root user "user" with ID 1000 (Required for HF Spaces)
+RUN useradd -m -u 1000 user
+
+# 3. Create necessary directories with correct permissions
+# We need a cache folder that the user can write to when downloading the model
+RUN mkdir -p /app/cache && \
+    mkdir -p /app/models && \
+    chown -R user:user /app
+
+# 4. Switch to the non-root user
+USER user
+
+# 5. Install Python dependencies
+# We copy requirements first to leverage Docker layer caching
+COPY --chown=user:user requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# 6. Copy the application code
+COPY --chown=user:user main.py .
+
+# 7. Expose the port
+EXPOSE 7860
+
+# 8. Run the application
+# host 0.0.0.0 is required for Docker networking
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]

README.md

@@ -0,0 +1,9 @@
+---
+title: My AI-Chat APi
+emoji: 🤥
+colorFrom: red
+colorTo: pink
+sdk: docker
+app_file: main.py
+pinned: false
+---

main.py

@@ -0,0 +1,112 @@
+import os
+import logging
+from contextlib import asynccontextmanager
+from fastapi import FastAPI, HTTPException, Request
+from pydantic import BaseModel, Field
+from typing import List, Optional
+from llama_cpp import Llama
+from huggingface_hub import hf_hub_download
+
+# --- 1. Logger Setup (Production Standard) ---
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+)
+logger = logging.getLogger("qwen-api")
+
+# --- 2. Global State ---
+model_instance: Optional[Llama] = None
+
+# Settings
+REPO_ID = "Qwen/Qwen2.5-1.5B-Instruct-GGUF"
+FILENAME = "qwen2.5-1.5b-instruct-q4_k_m.gguf"
+N_THREADS = int(os.getenv("CPU_THREADS", "2")) # Default to 2 for your hardware
+
+# --- 3. Lifespan (Startup/Shutdown Logic) ---
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    global model_instance
+    logger.info("STARTUP: Initializing Application...")
+    
+    try:
+        # Download model using huggingface_hub (More robust than curl)
+        logger.info(f"Downloading model {REPO_ID} -> {FILENAME}...")
+        model_path = hf_hub_download(
+            repo_id=REPO_ID,
+            filename=FILENAME,
+            local_dir="./models"
+        )
+        logger.info(f"Model downloaded to: {model_path}")
+
+        # Load Model
+        logger.info("Loading Llama model into memory...")
+        model_instance = Llama(
+            model_path=model_path,
+            n_ctx=4096,
+            n_threads=N_THREADS,
+            n_batch=512,
+            verbose=False 
+        )
+        logger.info("STARTUP: Model loaded successfully!")
+        
+    except Exception as e:
+        logger.error(f"CRITICAL: Failed to load model: {e}")
+        raise e
+        
+    yield
+    
+    # Shutdown logic (if needed)
+    logger.info("SHUTDOWN: Cleaning up resources...")
+    model_instance = None
+
+# --- 4. FastAPI App Definition ---
+app = FastAPI(title="Qwen Production API", version="1.0.0", lifespan=lifespan)
+
+# --- 5. Data Models ---
+class Message(BaseModel):
+    role: str
+    content: str
+
+class ChatCompletionRequest(BaseModel):
+    messages: List[Message]
+    temperature: Optional[float] = 0.7
+    max_tokens: Optional[int] = 512
+    stream: Optional[bool] = False
+
+# --- 6. Endpoints ---
+
+@app.get("/")
+async def root():
+    """Root endpoint to verify api is reachable"""
+    logger.info("Health check on root / accessed")
+    return {
+        "message": "Qwen 2.5 (1.5B) CPU Inference API is Running",
+        "docs_url": "/docs"
+    }
+
+@app.get("/ping")
+async def ping():
+    """Simple health check for monitoring tools"""
+    return "pong"
+
+@app.post("/v1/chat/completions")
+async def chat_completions(request: ChatCompletionRequest):
+    """OpenAI-compatible chat completion endpoint"""
+    if not model_instance:
+        logger.error("Request received but model not loaded")
+        raise HTTPException(status_code=503, detail="Model is not ready yet")
+
+    logger.info(f"Generating completion. Temp: {request.temperature}, MaxTokens: {request.max_tokens}")
+    
+    try:
+        # llama-cpp-python handles the chat formatting automatically
+        response = model_instance.create_chat_completion(
+            messages=[m.model_dump() for m in request.messages],
+            temperature=request.temperature,
+            max_tokens=request.max_tokens,
+            stream=request.stream
+        )
+        return response
+    except Exception as e:
+        logger.error(f"Inference Error: {str(e)}")
+        raise HTTPException(status_code=500, detail="Internal inference error")

requirements.txt

@@ -0,0 +1,5 @@
+fastapi>=0.115.0
+uvicorn>=0.30.0
+pydantic>=2.8.0
+llama-cpp-python>=0.2.90
+huggingface-hub>=0.24.0