public CUDA + llama.cpp inference base image

Dockerfile

@@ -0,0 +1,70 @@
+# syntax=docker/dockerfile:1.6
+# PUBLIC base image for GPU inference jobs (HF Jobs).
+#
+# Contains ONLY open-source, generic ML/runtime dependencies (CUDA, cu124 torch,
+# the precompiled llama.cpp server, ...) — NO proprietary code, and no names that
+# reveal which models/techniques the pipeline uses. The Job's bootstrap
+# (/opt/run-job.sh) pulls the private wheel + demo modules + the remaining runtime
+# dependencies at startup and runs run_job.py. Mirrors the old single-Space
+# Dockerfile's proven CUDA/torch/llama sequence.
+
+FROM ghcr.io/ggml-org/llama.cpp:server-cuda AS llama_bin
+
+FROM nvidia/cuda:12.8.1-cudnn-runtime-ubuntu24.04
+
+ENV DEBIAN_FRONTEND=noninteractive \
+    PIP_NO_CACHE_DIR=0 \
+    PYTHONUNBUFFERED=1 \
+    HF_HUB_ENABLE_HF_TRANSFER=1 \
+    LD_LIBRARY_PATH=/opt/llama.cpp/build/bin:/usr/local/cuda/lib64 \
+    PATH=/usr/local/cuda/bin:$PATH \
+    PIP_BREAK_SYSTEM_PACKAGES=1
+
+# libgomp1 is required by the per-microarch libggml-cpu-*.so variants from the
+# upstream server-cuda image (without it llama-server finds no CPU backend).
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        python3.12 python3.12-dev python3.12-venv python3-pip \
+        git curl ca-certificates \
+        libgomp1 \
+    && rm -rf /var/lib/apt/lists/*
+
+# Precompiled CUDA llama-server (binary + shared libs) from the upstream image.
+RUN mkdir -p /opt/llama.cpp/build/bin
+COPY --from=llama_bin /app/ /opt/llama.cpp/build/bin/
+RUN chmod +x /opt/llama.cpp/build/bin/llama-server
+
+WORKDIR /app
+
+# CUDA torch first, so the dependency install picks it up instead of the CPU wheel.
+RUN --mount=type=cache,target=/root/.cache/pip \
+    pip install --index-url https://download.pytorch.org/whl/cu124 torch
+
+# Install the generic DEPENDENCIES only. A stub package (empty
+# dataset_reviewer/__init__.py) lets `pip install .` resolve the deps listed in the
+# generated slim pyproject WITHOUT shipping any proprietary source into this public
+# image. At runtime the Job replaces the stub with the real wheel and installs the
+# remaining (technique-revealing) deps (see run-job.sh).
+COPY pyproject.toml /app/pyproject.toml
+COPY dataset_reviewer /app/dataset_reviewer
+RUN --mount=type=cache,target=/root/.cache/pip pip install /app
+
+# Explicit pins for the bootstrap + demo modules. huggingface_hub/python-dotenv are
+# already transitive; telethon is the Telegram client (MTProto) the Job uses to
+# resolve a submitter's @username and DM them — not a dataset_reviewer dep.
+RUN --mount=type=cache,target=/root/.cache/pip \
+    pip install "huggingface_hub[hf_transfer]" python-dotenv "telethon==1.43.2"
+
+COPY run-job.sh /opt/run-job.sh
+RUN chmod +x /opt/run-job.sh
+
+ENV LLAMA_CPP_DIR=/opt/llama.cpp \
+    LLAMA_SERVER_BIN=/opt/llama.cpp/build/bin/llama-server \
+    GGUF_DIR=/models \
+    LLM_PORT=8080 \
+    LLM_CTX=32768 \
+    LLM_PARALLEL=4
+
+# Default command keeps the Space itself idle+RUNNING (so the image publishes
+# cleanly); HF Jobs override this with `bash /opt/run-job.sh`.
+EXPOSE 7860
+CMD ["python3", "-m", "http.server", "7860"]

README.md

@@ -1,10 +1,18 @@
 ---
-title: Cuda Llm Inference Base
-emoji: 📈
-colorFrom: blue
-colorTo: blue
+title: CUDA LLM Inference Base
+emoji: 🧱
+colorFrom: gray
+colorTo: indigo
 sdk: docker
+app_port: 7860
 pinned: false
+short_description: CUDA + llama.cpp inference base image.
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+Public base image: CUDA + cu124 torch + a precompiled llama.cpp server + generic
+ML/runtime dependencies, for running GPU inference jobs on HF Jobs. It contains
+**no proprietary code**; application code and any remaining dependencies are
+installed at runtime from a private repo.
+
+This Space exists only to build + publish the image; it serves a trivial idle
+endpoint.

dataset_reviewer/__init__.py

@@ -0,0 +1,4 @@
+"""Stub package — present only so `pip install .` resolves dataset-reviewer's
+dependencies into this public base image WITHOUT shipping any proprietary source.
+The real dataset_reviewer wheel is force-reinstalled over this stub at Job runtime
+(see /opt/run-job.sh)."""

pyproject.toml

@@ -0,0 +1,47 @@
+[project]
+name = "dataset-reviewer"
+version = "0.1.0"
+requires-python = ">=3.12"
+dependencies = [
+    "datasets>=4.5.0",
+    "datatrove[io]>=0.9.0",
+    "duckdb>=1.4.4",
+    "ddgs",
+    "httpx",
+    "jinja2>=3.1.6",
+    "lancedb>=0.29.2",
+    "langchain>=1.2",
+    "langchain-community",
+    "langchain-huggingface",
+    "langchain-mcp-adapters",
+    "langchain-openai",
+    "langchain-tavily",
+    "openai>=1.0",
+    "markdownify",
+    "pyarrow>=23.0.1",
+    "pydantic>=2.12.5",
+    "markdown-it-py>=4.0.0",
+    "playwright>=1.58.0",
+    "rich>=14.3.3",
+    "langgraph",
+    "loguru",
+    "python-dotenv",
+    "huggingface-hub",
+    "pandas",
+    "requests",
+    "markupsafe",
+    "fsspec",
+    "transformers>=4.40",
+    "torch>=2.11.0",
+    "tiktoken>=0.12.0",
+    "tenacity>=9.1.4",
+    "pillow>=12.2.0",
+    "nh3>=0.2",
+]
+
+[build-system]
+requires = ["setuptools>=61"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.packages.find]
+include = ["dataset_reviewer*"]

run-job.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+# HF Job bootstrap: pull the private dataset_reviewer wheel + demo modules +
+# runtime requirements from JOB_CODE_REPO (authenticated by HF_TOKEN), install the
+# wheel over the baked stub, install the technique-revealing deps kept out of the
+# public image, then run the headless pipeline. Each phase is timed so its cost is
+# visible in the Job log. Keeps proprietary code + model/technique names OUT of the
+# public base image.
+set -euo pipefail
+
+: "${JOB_CODE_REPO:?JOB_CODE_REPO is required}"
+
+t=$SECONDS
+echo "[bootstrap] downloading job code from ${JOB_CODE_REPO}"
+python3 - <<'PY'
+import os
+from huggingface_hub import snapshot_download
+snapshot_download(repo_id=os.environ["JOB_CODE_REPO"], repo_type="model", local_dir="/code")
+PY
+echo "[bootstrap] job code downloaded in $((SECONDS - t))s"
+
+t=$SECONDS
+echo "[bootstrap] installing dataset_reviewer wheel over the baked stub"
+pip install --no-deps --force-reinstall --no-index /code/wheels/*.whl
+echo "[bootstrap] wheel installed in $((SECONDS - t))s"
+
+if [ -f /code/requirements-runtime.txt ]; then
+  t=$SECONDS
+  echo "[bootstrap] installing runtime ML dependencies"
+  pip install -r /code/requirements-runtime.txt
+  echo "[bootstrap] runtime deps installed in $((SECONDS - t))s"
+fi
+
+echo "[bootstrap] launching run_job.py"
+cd /code
+exec python3 -u run_job.py