Try qwen3

Dockerfile

@@ -1,37 +1,44 @@
 # Use an official Python runtime as a base image
 FROM python:3.10
 
+# Set non-interactive for apt
+ENV DEBIAN_FRONTEND=noninteractive \
+    # Rust toolchain dirs
+    RUSTUP_HOME=/root/.rustup \
+    CARGO_HOME=/root/.cargo \
+    PATH=/root/.cargo/bin:$PATH
+
 # Set working directory
 WORKDIR /app
 
-# Install system dependencies for llama_cpp
-RUN apt-get update && apt-get install -y \
-    gcc \
-    g++ \
-    libffi-dev \
-    libgcc-s1 \
-    libstdc++6 \
-    libopenblas-dev \
-    && rm -rf /var/lib/apt/lists/*
-
-# Set environment variables for cache
-ENV TRANSFORMERS_CACHE=/app/cache
-ENV HF_HOME=/app/cache
-
-# Create cache directory with write permissions
+# Install system dependencies, Rust, and build tools
+RUN apt-get update && apt-get install -y --no-install-recommends \
+      build-essential cmake git curl wget ninja-build libgomp1 ca-certificates \
+      gcc g++ libffi-dev libgcc-s1 libstdc++6 libopenblas-dev \
+    && rm -rf /var/lib/apt/lists/* \
+    && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y \
+    && rustup default stable
+
+# Prepare cache directory
 RUN mkdir -p /app/cache && chmod -R 777 /app/cache
 
-# Copy and install requirements
+# Copy and install Python requirements (excluding llama-cpp-python)
 COPY requirements.txt .
-RUN pip install --no-cache-dir -r requirements.txt
+RUN sed -i '/llama-cpp-python/d' requirements.txt \
+ && pip install --no-cache-dir -r requirements.txt
+
+# Clone and build llama-cpp-python from source
+RUN git clone --recurse-submodules https://github.com/abetlen/llama-cpp-python.git /tmp/llama-cpp-python \
+ && cd /tmp/llama-cpp-python \
+ # Force a CMake rebuild even if cached
+ && FORCE_CMAKE=1 pip install --no-cache-dir -e . \
+ && rm -rf /tmp/llama-cpp-python
 
-# Copy application files
-COPY app.py .
-COPY cv_embeddings.json .
-COPY cv_text.txt .
+# Copy application code and data
+COPY app.py cv_embeddings.json cv_text.txt ./
 
-# Expose port
+# Expose the port your FastAPI app runs on
 EXPOSE 7860
 
-# Run the application
+# Launch
 CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]

app.py

@@ -32,8 +32,8 @@ login(token=hf_token)
 
 # Models Configuration
 sentence_transformer_model = "all-MiniLM-L6-v2"
-repo_id = "bartowski/deepcogito_cogito-v1-preview-llama-3B-GGUF" # "bartowski/deepcogito_cogito-v1-preview-llama-8B-GGUF"
-filename = "deepcogito_cogito-v1-preview-llama-3B-Q4_K_M.gguf"
+repo_id = "unsloth/Qwen3-1.7B-GGUF" # "bartowski/deepcogito_cogito-v1-preview-llama-3B-GGUF" # "bartowski/deepcogito_cogito-v1-preview-llama-8B-GGUF"
+filename = "Qwen3-1.7B-Q4_K_M.gguf" # "deepcogito_cogito-v1-preview-llama-3B-Q4_K_M.gguf"
 
 # Define FAQs
 faqs = [