public base image (hf-image-estimate)

Dockerfile

@@ -1,9 +1,9 @@
 # syntax=docker/dockerfile:1.6
 # PUBLIC base image for the cheap CPU ESTIMATE job (HF Jobs) — slim, no GPU stack.
 #
-# Same contract as demo/image/Dockerfile: ONLY open-source, generic dependencies — NO proprietary
-# code, and no names that reveal which models/techniques the pipeline uses. The Job's bootstrap
-# (/opt/run-job.sh) pulls the private wheel + demo modules at startup and runs run_job.py.
+# Same contract as deploy/hf-image-llama-cpp/Dockerfile: ONLY open-source, generic dependencies — NO
+# proprietary code, and no names that reveal which models/techniques the pipeline uses. The Job's
+# bootstrap (/opt/run-job.sh) pulls the private wheel + job modules at startup and runs run_job.py.
 #
 # This variant is CPU-ONLY and slim: it omits CUDA/torch and the GPU inference servers (the estimate
 # runs zero signals and starts no model server — run_job.py returns at the ESTIMATE_ONLY branch
@@ -12,10 +12,10 @@
 # estimate's wall-clock. A lightweight tokenizer loads via the Rust `tokenizers` backend, so no
 # torch is needed.
 #
-# Build: JOB_IMAGE_DIR=image-estimate JOB_IMAGE_BASE_PROVIDES=torch \
-#        JOB_IMAGE_SPACE=<owner>/cpu-estimate-base make demo-deploy-job
-# (run-job.sh + the dataset_reviewer stub are shared from demo/image/; the omitted dep group is
-#  applied automatically for this variant — see demo/deploy_job.py.)
+# Build: HF_JOB_IMAGE_DIR=hf-image-estimate HF_JOB_IMAGE_BASE_PROVIDES=torch \
+#        HF_JOB_IMAGE_SPACE=<owner>/cpu-estimate-base make deploy-hf-job
+# (run-job.sh + the dataset_reviewer stub are shared from deploy/hf-image-common/; the omitted dep
+#  group is applied automatically for this variant — see deploy/deploy_hf_job.py.)
 
 FROM python:3.12-slim
 
@@ -33,7 +33,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 WORKDIR /app
 
 # Install the generic DEPENDENCIES only — the generated pyproject omits torch
-# (JOB_IMAGE_BASE_PROVIDES=torch) since the estimate needs no torch/CUDA. A stub package
+# (HF_JOB_IMAGE_BASE_PROVIDES=torch) since the estimate needs no torch/CUDA. A stub package
 # (empty dataset_reviewer/__init__.py) lets `pip install .` resolve the deps WITHOUT shipping
 # proprietary source into this public image. At runtime the Job replaces the stub with the
 # real wheel; run-job.sh skips the technique-revealing runtime deps for ESTIMATE_ONLY (never
@@ -42,11 +42,10 @@ COPY pyproject.toml /app/pyproject.toml
 COPY dataset_reviewer /app/dataset_reviewer
 RUN --mount=type=cache,target=/root/.cache/pip pip install /app
 
-# Explicit pins for the bootstrap + demo modules (transitive above, pinned so the bootstrap
-# never depends on resolution order). telethon is the Telegram client the Job imports — not a
-# dataset_reviewer dep.
-RUN --mount=type=cache,target=/root/.cache/pip \
-    pip install "huggingface_hub[hf_transfer]" python-dotenv "telethon==1.43.2"
+# Non-library job deps, single-sourced in job_deps.JOB_EXTRA_DEPS — deploy_hf_job.py
+# stages the requirements file into this build context.
+COPY requirements-extras.txt /app/requirements-extras.txt
+RUN --mount=type=cache,target=/root/.cache/pip pip install -r /app/requirements-extras.txt
 
 COPY run-job.sh /opt/run-job.sh
 RUN chmod +x /opt/run-job.sh

pyproject.toml

@@ -4,7 +4,7 @@ version = "0.1.0"
 requires-python = ">=3.12"
 dependencies = [
     "datasets>=5.0.0",
-    "datatrove[io]>=0.9.0",
+    "datatrove[io] @ git+https://github.com/huggingface/datatrove.git@a035d36e8a13d22ce04e84b84f30b4ec45f9c48e",
     "duckdb>=1.4.4",
     "httpx",
     "jinja2>=3.1.6",
@@ -16,7 +16,7 @@ dependencies = [
     "rich>=14.3.3",
     "loguru",
     "python-dotenv",
-    "huggingface-hub",
+    "huggingface-hub>=1.13,<1.17",
     "pandas",
     "pytz",
     "requests",

requirements-extras.txt

@@ -0,0 +1,2 @@
+huggingface_hub[hf_transfer]
+telethon==1.43.2

run-job.sh

@@ -1,20 +1,23 @@
 #!/usr/bin/env bash
-# HF Job bootstrap: pull the private dataset_reviewer wheel + demo modules +
-# runtime requirements from JOB_CODE_REPO (authenticated by HF_TOKEN), install the
-# wheel over the baked stub, install the technique-revealing deps kept out of the
-# public image, then run the headless pipeline. Each phase is timed so its cost is
-# visible in the Job log. Keeps proprietary code + model/technique names OUT of the
-# public base image.
+# HF Job bootstrap: pull the private dataset_reviewer wheel + job modules +
+# runtime requirements from HF_JOB_CODE_REPO (authenticated by HF_TOKEN), install
+# the wheel over the baked stub, install the technique-revealing deps kept out of
+# the public image, then run the headless pipeline. Each phase is timed so its cost
+# is visible in the Job log. Keeps proprietary code + model/technique names OUT of
+# the public base image.
 set -euo pipefail
 
-: "${JOB_CODE_REPO:?JOB_CODE_REPO is required}"
+# JOB_CODE_REPO is the pre-rename env name; launchers older than the rename send only it.
+: "${HF_JOB_CODE_REPO:=${JOB_CODE_REPO:-}}"
+: "${HF_JOB_CODE_REPO:?HF_JOB_CODE_REPO is required}"
+export HF_JOB_CODE_REPO
 
 t=$SECONDS
-echo "[bootstrap] downloading job code from ${JOB_CODE_REPO}"
+echo "[bootstrap] downloading job code from ${HF_JOB_CODE_REPO}"
 python3 - <<'PY'
 import os
 from huggingface_hub import snapshot_download
-snapshot_download(repo_id=os.environ["JOB_CODE_REPO"], repo_type="model", local_dir="/code")
+snapshot_download(repo_id=os.environ["HF_JOB_CODE_REPO"], repo_type="model", local_dir="/code")
 PY
 echo "[bootstrap] job code downloaded in $((SECONDS - t))s"