init commit

.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+    "python-envs.defaultEnvManager": "ms-python.python:conda",
+    "python-envs.defaultPackageManager": "ms-python.python:conda",
+    "python-envs.pythonProjects": []
+}

Dockerfile

@@ -0,0 +1,22 @@
+FROM ollama/ollama
+
+# 🔴 IMPORTANT: remove ollama entrypoint
+ENTRYPOINT []
+
+WORKDIR /app
+
+RUN apt-get update && \
+    apt-get install -y python3-venv python3-pip build-essential libffi-dev libssl-dev && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN python3 -m venv /opt/venv
+ENV PATH="/opt/venv/bin:$PATH"
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . .
+
+EXPOSE 8000
+
+CMD sh -c "ollama serve & sleep 5 && ollama pull qwen2.5:1.5b && uvicorn app:app --host 0.0.0.0 --port $PORT"

app.py

@@ -0,0 +1,48 @@
+from fastapi import FastAPI, Request
+from fastapi.responses import StreamingResponse
+import httpx
+import json
+import asyncio
+
+app = FastAPI()
+
+
+OLLAMA_URL = "http://localhost:11434/api/chat"
+# OLLAMA_URL = "http://localhost:11434/api/generate"
+
+@app.post("/v1/chat/completions")
+async def chat(req: Request):
+    body = await req.json()
+    messages = body.get("messages", [])
+    
+    payload = {
+        "model": "qwen2.5:1.5b",
+        "messages": messages,
+        "stream": True,
+        "options": {
+            "temperature": 0.8,
+            "top_p": 0.9,
+        }
+    }
+
+    async def event_stream():
+        
+        async with httpx.AsyncClient(timeout=None) as client:
+            async with client.stream("POST", OLLAMA_URL, json=payload) as response:
+                
+                async for line in response.aiter_lines():
+                    if not line:
+                        continue
+                    try:
+                        chunk = json.loads(line)
+                        if 'message' in chunk and 'content' in chunk['message']:
+                            content = chunk['message']['content']
+                            
+                            yield content
+                        
+                        if chunk.get("done"):
+                            break
+                    except json.JSONDecodeError:
+                        continue
+
+    return StreamingResponse(event_stream(), media_type="text/plain")

requirements.txt

@@ -0,0 +1,3 @@
+fastapi
+uvicorn
+httpx