Upload 5 files

Dockerfile

@@ -1,44 +1,44 @@
-FROM nvidia/cuda:12.4.1-cudnn-devel-ubuntu22.04
-
-ENV DEBIAN_FRONTEND=noninteractive
-ENV PYTHONUNBUFFERED=1
-ENV PORT=7860
-ENV LLAMA_PORT=8000
-ENV CMAKE_BUILD_PARALLEL_LEVEL=2
-
-RUN apt-get update && apt-get install -y \
-    git \
-    build-essential \
-    cmake \
-    curl \
-    python3 \
-    python3-pip \
-    ca-certificates \
-    && rm -rf /var/lib/apt/lists/*
-
-WORKDIR /app
-
-RUN git clone --depth 1 https://github.com/ggml-org/llama.cpp /app/llama.cpp && \
-    cmake -S /app/llama.cpp -B /app/llama.cpp/build \
-      -DCMAKE_BUILD_TYPE=Release \
-      -DGGML_CUDA=ON \
-      -DLLAMA_BUILD_COMMON=ON \
-      -DLLAMA_BUILD_TOOLS=ON \
-      -DLLAMA_BUILD_SERVER=ON \
-      -DLLAMA_BUILD_EXAMPLES=OFF \
-      -DLLAMA_BUILD_TESTS=OFF \
-      -DLLAMA_BUILD_WEBUI=OFF \
-      -DBUILD_SHARED_LIBS=OFF && \
-    cmake --build /app/llama.cpp/build --config Release --target llama-server -j2
-
-COPY requirements.txt /app/requirements.txt
-RUN pip3 install --no-cache-dir -r /app/requirements.txt
-
-COPY app.py /app/app.py
-COPY start.sh /app/start.sh
-
-RUN chmod +x /app/start.sh
-
-EXPOSE 7860
-
+FROM nvidia/cuda:12.4.1-cudnn-devel-ubuntu22.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV PYTHONUNBUFFERED=1
+ENV PORT=7860
+ENV LLAMA_PORT=8000
+ENV CMAKE_BUILD_PARALLEL_LEVEL=2
+
+RUN apt-get update && apt-get install -y \
+    git \
+    build-essential \
+    cmake \
+    curl \
+    python3 \
+    python3-pip \
+    ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+RUN git clone --depth 1 https://github.com/ggml-org/llama.cpp /app/llama.cpp && \
+    cmake -S /app/llama.cpp -B /app/llama.cpp/build \
+      -DCMAKE_BUILD_TYPE=Release \
+      -DGGML_CUDA=ON \
+      -DLLAMA_BUILD_COMMON=ON \
+      -DLLAMA_BUILD_TOOLS=ON \
+      -DLLAMA_BUILD_SERVER=ON \
+      -DLLAMA_BUILD_EXAMPLES=OFF \
+      -DLLAMA_BUILD_TESTS=OFF \
+      -DLLAMA_BUILD_WEBUI=OFF \
+      -DBUILD_SHARED_LIBS=OFF && \
+    cmake --build /app/llama.cpp/build --config Release --target llama-server -j2
+
+COPY requirements.txt /app/requirements.txt
+RUN pip3 install --no-cache-dir -r /app/requirements.txt
+
+COPY app.py /app/app.py
+COPY start.sh /app/start.sh
+
+RUN sed -i 's/\r$//' /app/start.sh && chmod +x /app/start.sh
+
+EXPOSE 7860
+
 CMD ["/app/start.sh"]

README.md

@@ -1,45 +1,45 @@
----
-title: Gemma 4 31B OpenAI API
-emoji: 🤖
-colorFrom: indigo
-colorTo: blue
-sdk: docker
-app_port: 7860
-startup_duration_timeout: 1h
-suggested_hardware: l40sx1
-models:
-  - llmfan46/gemma-4-31B-it-uncensored-heretic-GGUF
-tags:
-  - gguf
-  - llama.cpp
-  - openai-compatible
-  - gemma
-  - multimodal
-pinned: false
----
-
-# Gemma 4 31B OpenAI API
-
-OpenAI-compatible API for Gemma 4 31B GGUF running on llama.cpp in a Hugging Face Docker Space.
-
-## Endpoints
-
-- `/`
-- `/health`
-- `/v1/chat/completions`
-
-## Environment Variables
-
-- `MODEL_REPO=llmfan46/gemma-4-31B-it-uncensored-heretic-GGUF`
-- `MODEL_FILE=gemma-4-31B-it-uncensored-heretic-Q8_0.gguf`
-- `MMPROJ_FILE=gemma-4-31B-it-mmproj-BF16.gguf`
-- `CTX_SIZE=4096`
-- `N_GPU_LAYERS=-1`
-
-## Notes
-
-This Space is configured as a Docker Space and listens on port `7860`.
-
-`startup_duration_timeout: 1h` gives the container more time to become healthy during large model startup.
-
+---
+title: Gemma 4 31B OpenAI API
+emoji: 🤖
+colorFrom: indigo
+colorTo: blue
+sdk: docker
+app_port: 7860
+startup_duration_timeout: 1h
+suggested_hardware: l40sx1
+models:
+  - llmfan46/gemma-4-31B-it-uncensored-heretic-GGUF
+tags:
+  - gguf
+  - llama.cpp
+  - openai-compatible
+  - gemma
+  - multimodal
+pinned: false
+---
+
+# Gemma 4 31B OpenAI API
+
+OpenAI-compatible API for Gemma 4 31B GGUF running on llama.cpp in a Hugging Face Docker Space.
+
+## Endpoints
+
+- `/`
+- `/health`
+- `/v1/chat/completions`
+
+## Environment Variables
+
+- `MODEL_REPO=llmfan46/gemma-4-31B-it-uncensored-heretic-GGUF`
+- `MODEL_FILE=gemma-4-31B-it-uncensored-heretic-Q8_0.gguf`
+- `MMPROJ_FILE=gemma-4-31B-it-mmproj-BF16.gguf`
+- `CTX_SIZE=4096`
+- `N_GPU_LAYERS=-1`
+
+## Notes
+
+This Space is configured as a Docker Space and listens on port `7860`.
+
+`startup_duration_timeout: 1h` gives the container more time to become healthy during large model startup.
+
 The app is intended for an `l40sx1` hardware flavor.

app.py

@@ -1,45 +1,45 @@
-from fastapi import FastAPI, Request, Response
-import httpx
-
-app = FastAPI()
-BACKEND = "http://127.0.0.1:8000"
-
-@app.get("/")
-async def root():
-    return {
-        "ok": True,
-        "message": "GGUF OpenAI-compatible API is running",
-        "chat_completions": "/v1/chat/completions"
-    }
-
-@app.get("/health")
-async def health():
-    async with httpx.AsyncClient(timeout=10) as client:
-        r = await client.get(f"{BACKEND}/health")
-        return Response(
-            content=r.content,
-            status_code=r.status_code,
-            media_type=r.headers.get("content-type", "application/json"),
-        )
-
-@app.api_route("/v1/{path:path}", methods=["GET", "POST", "PUT", "PATCH", "DELETE"])
-async def proxy(path: str, request: Request):
-    url = f"{BACKEND}/v1/{path}"
-    body = await request.body()
-    headers = dict(request.headers)
-    headers.pop("host", None)
-
-    async with httpx.AsyncClient(timeout=None) as client:
-        r = await client.request(
-            method=request.method,
-            url=url,
-            headers=headers,
-            content=body,
-            params=request.query_params,
-        )
-
-    return Response(
-        content=r.content,
-        status_code=r.status_code,
-        media_type=r.headers.get("content-type", "application/json"),
+from fastapi import FastAPI, Request, Response
+import httpx
+
+app = FastAPI()
+BACKEND = "http://127.0.0.1:8000"
+
+@app.get("/")
+async def root():
+    return {
+        "ok": True,
+        "message": "GGUF OpenAI-compatible API is running",
+        "chat_completions": "/v1/chat/completions"
+    }
+
+@app.get("/health")
+async def health():
+    async with httpx.AsyncClient(timeout=10) as client:
+        r = await client.get(f"{BACKEND}/health")
+        return Response(
+            content=r.content,
+            status_code=r.status_code,
+            media_type=r.headers.get("content-type", "application/json"),
+        )
+
+@app.api_route("/v1/{path:path}", methods=["GET", "POST", "PUT", "PATCH", "DELETE"])
+async def proxy(path: str, request: Request):
+    url = f"{BACKEND}/v1/{path}"
+    body = await request.body()
+    headers = dict(request.headers)
+    headers.pop("host", None)
+
+    async with httpx.AsyncClient(timeout=None) as client:
+        r = await client.request(
+            method=request.method,
+            url=url,
+            headers=headers,
+            content=body,
+            params=request.query_params,
+        )
+
+    return Response(
+        content=r.content,
+        status_code=r.status_code,
+        media_type=r.headers.get("content-type", "application/json"),
     )

requirements.txt

@@ -1,4 +1,4 @@
-fastapi
-uvicorn
-httpx
+fastapi
+uvicorn
+httpx
 huggingface_hub[cli]

start.sh

@@ -1,35 +1,35 @@
-#!/bin/bash
-set -e
-
-MODEL_REPO="${MODEL_REPO:-llmfan46/gemma-4-31B-it-uncensored-heretic-GGUF}"
-MODEL_FILE="${MODEL_FILE:-gemma-4-31B-it-uncensored-heretic-Q8_0.gguf}"
-MMPROJ_FILE="${MMPROJ_FILE:-gemma-4-31B-it-mmproj-BF16.gguf}"
-CTX_SIZE="${CTX_SIZE:-4096}"
-PORT="${PORT:-7860}"
-LLAMA_PORT="${LLAMA_PORT:-8000}"
-N_GPU_LAYERS="${N_GPU_LAYERS:--1}"
-HF_HOME="${HF_HOME:-/data/huggingface}"
-LLAMA_CACHE="${LLAMA_CACHE:-/data/llama-cache}"
-
-mkdir -p "${HF_HOME}" "${LLAMA_CACHE}" /data/models
-export HF_HOME
-export LLAMA_CACHE
-
-python3 -m pip install --no-cache-dir -U "huggingface_hub[cli]"
-
-huggingface-cli download "${MODEL_REPO}" "${MODEL_FILE}" --local-dir /data/models
-huggingface-cli download "${MODEL_REPO}" "${MMPROJ_FILE}" --local-dir /data/models
-
-MODEL_PATH="/data/models/${MODEL_FILE}"
-MMPROJ_PATH="/data/models/${MMPROJ_FILE}"
-
-/app/llama.cpp/build/bin/llama-server \
-  -m "${MODEL_PATH}" \
-  --mmproj "${MMPROJ_PATH}" \
-  --host 0.0.0.0 \
-  --port "${LLAMA_PORT}" \
-  -c "${CTX_SIZE}" \
-  -ngl "${N_GPU_LAYERS}" \
-  &
-
+#!/bin/bash
+set -e
+
+MODEL_REPO="${MODEL_REPO:-llmfan46/gemma-4-31B-it-uncensored-heretic-GGUF}"
+MODEL_FILE="${MODEL_FILE:-gemma-4-31B-it-uncensored-heretic-Q8_0.gguf}"
+MMPROJ_FILE="${MMPROJ_FILE:-gemma-4-31B-it-mmproj-BF16.gguf}"
+CTX_SIZE="${CTX_SIZE:-4096}"
+PORT="${PORT:-7860}"
+LLAMA_PORT="${LLAMA_PORT:-8000}"
+N_GPU_LAYERS="${N_GPU_LAYERS:--1}"
+HF_HOME="${HF_HOME:-/data/huggingface}"
+LLAMA_CACHE="${LLAMA_CACHE:-/data/llama-cache}"
+
+mkdir -p "${HF_HOME}" "${LLAMA_CACHE}" /data/models
+export HF_HOME
+export LLAMA_CACHE
+
+python3 -m pip install --no-cache-dir -U "huggingface_hub[cli]"
+
+huggingface-cli download "${MODEL_REPO}" "${MODEL_FILE}" --local-dir /data/models
+huggingface-cli download "${MODEL_REPO}" "${MMPROJ_FILE}" --local-dir /data/models
+
+MODEL_PATH="/data/models/${MODEL_FILE}"
+MMPROJ_PATH="/data/models/${MMPROJ_FILE}"
+
+/app/llama.cpp/build/bin/llama-server \
+  -m "${MODEL_PATH}" \
+  --mmproj "${MMPROJ_PATH}" \
+  --host 0.0.0.0 \
+  --port "${LLAMA_PORT}" \
+  -c "${CTX_SIZE}" \
+  -ngl "${N_GPU_LAYERS}" \
+  &
+
 exec uvicorn app:app --host 0.0.0.0 --port "${PORT}"