Makefile

.PHONY: install install-dev run run-local build-image fixtures test verify clean

VENV := .venv
PY := $(VENV)/bin/python
PORT ?= 7860
JOB_IMAGE ?= cc-repackage-job:dev
OUT_DIR ?= $(PWD)/.verify-out

# --- setup -------------------------------------------------------------------

# Install the Space runtime dependencies into a local venv (created if missing).
install:
	@test -d $(VENV) && echo "Using existing venv $(VENV)" || uv venv $(VENV) --python 3.12
	uv pip install --python $(VENV) -r requirements.txt

# Also install the tools needed to build fixtures and run tests/verification locally
# (duckdb, warcio, pyarrow, pytest, gradio_client). cdx_toolkit is NOT needed locally:
# the estimate job's SQL is vendored (src/index_query.py) and the fetch job installs
# cdxt itself inside its container (Dockerfile.job).
install-dev: install
	uv pip install --python $(VENV) pytest pandas gradio_client warcio pyarrow duckdb

# --- run ---------------------------------------------------------------------

# Run the app locally against real HF Jobs (requires HF login / Pro plan).
run:
	GRADIO_SERVER_PORT=$(PORT) $(PY) app.py

# Run the app in local podman mode against the toy fixtures (no HF/AWS, no cost).
run-local: fixtures build-image
	rm -rf $(OUT_DIR) && mkdir -p $(OUT_DIR)
	@echo "Starting Range server on :8080 and the app on :$(PORT) (Ctrl-C to stop)"
	@$(PY) scripts/range_server.py tests/fixtures/cc 8080 & echo $$! > $(OUT_DIR)/.srv.pid; \
	trap 'kill `cat $(OUT_DIR)/.srv.pid` 2>/dev/null' EXIT; \
	CC_REPACKAGE_EXECUTOR=podman \
	CC_PODMAN_CC_DIR=$(PWD)/tests/fixtures/cc \
	CC_PODMAN_OUT_DIR=$(OUT_DIR) \
	CC_PODMAN_IMAGE=$(JOB_IMAGE) \
	CC_PODMAN_SKIP_PIP=1 \
	GRADIO_SERVER_PORT=$(PORT) $(PY) app.py

# --- helpers -----------------------------------------------------------------

# Build the podman job image (cdx_toolkit preinstalled) used by local mode.
build-image:
	@podman image exists $(JOB_IMAGE) || podman build -f Dockerfile.job -t $(JOB_IMAGE) .

# Generate the toy CC bucket fixtures (index + WARC).
fixtures:
	$(PY) tests/make_fixtures.py tests/fixtures/cc

test:
	$(PY) -m pytest -q

# Full end-to-end local verification via podman (toy fixtures, no network).
verify:
	bash scripts/verify_local.sh

# Real-path verification against the LIVE commoncrawl bucket (needs HF_TOKEN).
verify-real:
	bash scripts/verify_real.sh

clean:
	rm -rf .verify-out* tests/fixtures/cc .pytest_cache
	find . -name __pycache__ -type d -prune -exec rm -rf {} +