Spaces:
Sleeping
Sleeping
File size: 2,479 Bytes
15eb4b9 8eaca44 15eb4b9 8eaca44 15eb4b9 9630bf6 15eb4b9 9630bf6 15eb4b9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 | .PHONY: install install-dev run run-local build-image fixtures test verify clean
VENV := .venv
PY := $(VENV)/bin/python
PORT ?= 7860
JOB_IMAGE ?= cc-repackage-job:dev
OUT_DIR ?= $(PWD)/.verify-out
# --- setup -------------------------------------------------------------------
# Install the Space runtime dependencies into a local venv (created if missing).
install:
@test -d $(VENV) && echo "Using existing venv $(VENV)" || uv venv $(VENV) --python 3.12
uv pip install --python $(VENV) -r requirements.txt
# Also install the tools needed to build fixtures and run tests/verification locally
# (duckdb, warcio, pyarrow, pytest, gradio_client). cdx_toolkit is NOT needed locally:
# the estimate job's SQL is vendored (src/index_query.py) and the fetch job installs
# cdxt itself inside its container (Dockerfile.job).
install-dev: install
uv pip install --python $(VENV) pytest pandas gradio_client warcio pyarrow duckdb
# --- run ---------------------------------------------------------------------
# Run the app locally against real HF Jobs (requires HF login / Pro plan).
run:
GRADIO_SERVER_PORT=$(PORT) $(PY) app.py
# Run the app in local podman mode against the toy fixtures (no HF/AWS, no cost).
run-local: fixtures build-image
rm -rf $(OUT_DIR) && mkdir -p $(OUT_DIR)
@echo "Starting Range server on :8080 and the app on :$(PORT) (Ctrl-C to stop)"
@$(PY) scripts/range_server.py tests/fixtures/cc 8080 & echo $$! > $(OUT_DIR)/.srv.pid; \
trap 'kill `cat $(OUT_DIR)/.srv.pid` 2>/dev/null' EXIT; \
CC_REPACKAGE_EXECUTOR=podman \
CC_PODMAN_CC_DIR=$(PWD)/tests/fixtures/cc \
CC_PODMAN_OUT_DIR=$(OUT_DIR) \
CC_PODMAN_IMAGE=$(JOB_IMAGE) \
CC_PODMAN_SKIP_PIP=1 \
GRADIO_SERVER_PORT=$(PORT) $(PY) app.py
# --- helpers -----------------------------------------------------------------
# Build the podman job image (cdx_toolkit preinstalled) used by local mode.
build-image:
@podman image exists $(JOB_IMAGE) || podman build -f Dockerfile.job -t $(JOB_IMAGE) .
# Generate the toy CC bucket fixtures (index + WARC).
fixtures:
$(PY) tests/make_fixtures.py tests/fixtures/cc
test:
$(PY) -m pytest -q
# Full end-to-end local verification via podman (toy fixtures, no network).
verify:
bash scripts/verify_local.sh
# Real-path verification against the LIVE commoncrawl bucket (needs HF_TOKEN).
verify-real:
bash scripts/verify_real.sh
clean:
rm -rf .verify-out* tests/fixtures/cc .pytest_cache
find . -name __pycache__ -type d -prune -exec rm -rf {} +
|