Initial setup

Dockerfile

@@ -0,0 +1,28 @@
+# Use an official Python runtime as a base image
+FROM python:3.10-slim
+
+# Set working directory
+WORKDIR /app
+
+# Install system dependencies (e.g., for torch, sentence-transformers)
+RUN apt-get update && apt-get install -y \
+    gcc \
+    g++ \
+    libffi-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy requirements file
+COPY requirements.txt .
+
+# Install Python dependencies
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy application files
+COPY app.py .
+COPY cv_embeddings.json .
+
+# Expose the port FastAPI will run on
+EXPOSE 7860
+
+# Command to run the application
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]

app.py

@@ -0,0 +1,67 @@
+import json
+import numpy as np
+from sentence_transformers import SentenceTransformer
+from transformers import pipeline, TextIteratorStreamer
+from threading import Thread
+import torch
+import torch.nn.functional as F
+from fastapi import FastAPI, HTTPException
+from fastapi.responses import StreamingResponse
+from pydantic import BaseModel
+
+app = FastAPI()
+
+# Load precomputed CV embeddings
+with open("cv_embeddings.json", "r", encoding="utf-8") as f:
+    cv_data = json.load(f)
+    cv_chunks = [item["chunk"] for item in cv_data]
+    cv_embeddings = np.array([item["embedding"] for item in cv_data])
+
+cv_embeddings_tensor = torch.tensor(cv_embeddings)
+
+embedder = SentenceTransformer("all-MiniLM-L6-v2", device="cpu")
+
+generator = pipeline(
+    "text-generation",
+    model="distilgpt2",
+    device=-1,
+)
+
+def retrieve_context(query, top_k=3):
+    query_embedding = embedder.encode(query, convert_to_tensor=True).unsqueeze(0)
+    similarities = F.cosine_similarity(query_embedding, cv_embeddings_tensor, dim=1)
+    top_k = min(top_k, len(similarities))
+    top_indices = torch.topk(similarities, k=top_k).indices.cpu().numpy()
+    return "\n".join([cv_chunks[i] for i in top_indices])
+
+def stream_response(query):
+    context = retrieve_context(query)
+    prompt = (
+        f"I am Tim Luka Horstmann, a German Computer Scientist. Based on my CV:\n{context}\n\n"
+        f"Question: {query}\nAnswer:"
+    )
+    
+    streamer = TextIteratorStreamer(generator.tokenizer, skip_prompt=True, skip_special_tokens=True)
+    generation_kwargs = {
+        "text_inputs": prompt,
+        "max_new_tokens": 200,
+        "do_sample": False,
+        "streamer": streamer,
+    }
+    
+    thread = Thread(target=generator, kwargs=generation_kwargs)
+    thread.start()
+    
+    for token in streamer:
+        yield f"data: {token}\n\n"
+    yield "data: [DONE]\n\n"
+
+class QueryRequest(BaseModel):
+    data: list
+
+@app.post("/api/predict")
+async def predict(request: QueryRequest):
+    if not request.data or not isinstance(request.data, list) or len(request.data) < 1:
+        raise HTTPException(status_code=400, detail="Invalid input: 'data' must be a non-empty list")
+    query = request.data[0]
+    return StreamingResponse(stream_response(query), media_type="text/event-stream")

requirements.txt

@@ -0,0 +1,6 @@
+fastapi==0.115.0
+uvicorn==0.31.0
+sentence-transformers==3.1.1
+transformers==4.44.2
+torch==2.4.1
+numpy==1.26.4