Upload 3 files

Dockerfile

@@ -0,0 +1,29 @@
+# Use the official Python image with a specific version
+FROM python:3.12.5-slim
+
+# Set the working directory in the container
+WORKDIR /app
+
+# Install system dependencies required for building Python packages
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    gcc \
+    g++ \
+    cmake \
+    git \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy the requirements file into the container
+COPY requirements.txt .
+
+# Install dependencies
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy the application code into the container
+COPY . .
+
+# Expose the application port
+EXPOSE 8000
+
+# Define the command to run the application
+CMD ["uvicorn", "llm:app", "--host", "0.0.0.0", "--port", "8000"]

llm.py

@@ -0,0 +1,38 @@
+from fastapi import FastAPI, HTTPException
+from pydantic import BaseModel
+from llama_cpp import Llama
+from huggingface_hub import hf_hub_download
+
+
+hf_hub_download(
+    repo_id="hugging-quants/Llama-3.2-1B-Instruct-Q4_K_M-GGUF",
+    filename="llama-3.2-1b-instruct-q4_k_m.gguf",
+    local_dir="./models"
+)
+
+# Initialize the LLM once when the application starts
+llm = Llama(
+    model_path=f"models/llama-3.2-1b-instruct-q4_k_m.gguf"
+)
+
+app = FastAPI()
+
+class ChatRequest(BaseModel):
+    message: str
+
+@app.post("/chat")
+async def chat_completion(request: ChatRequest):
+    try:
+        response = llm.create_chat_completion(
+            messages=[
+                {"role": "user", "content": request.message}
+            ]
+        )
+        return {
+            "response": response['choices'][0]['message']['content']
+        }
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+# To run the application:
+# uvicorn filename:app --reload

requirements.txt

@@ -0,0 +1,27 @@
+annotated-types==0.7.0
+anyio==4.6.2.post1
+certifi==2024.8.30
+charset-normalizer==3.4.0
+click==8.1.7
+diskcache==5.6.3
+fastapi==0.115.5
+filelock==3.16.1
+fsspec==2024.10.0
+h11==0.14.0
+huggingface-hub==0.26.2
+idna==3.10
+Jinja2==3.1.4
+llama_cpp_python==0.3.2
+MarkupSafe==3.0.2
+numpy==2.1.3
+packaging==24.2
+pydantic==2.9.2
+pydantic_core==2.23.4
+PyYAML==6.0.2
+requests==2.32.3
+sniffio==1.3.1
+starlette==0.41.2
+tqdm==4.67.0
+typing_extensions==4.12.2
+urllib3==2.2.3
+uvicorn==0.32.0