#!/usr/bin/env bash # End-to-end local verification with podman (no HF/AWS, no cost). # 1. unit tests # 2. (re)build toy fixtures # 3. build the job image (cdx_toolkit preinstalled) if missing # 4. start a Range-capable HTTP server for the fixture WARCs # 5. drive the app's Estimate + Launch buttons through the real podman runner # 6. validate the produced WARC with warcio set -euo pipefail cd "$(dirname "$0")/.." PY=.venv/bin/python PORT=8080 OUT_DIR="$PWD/.verify-out" IMAGE=cc-repackage-job:dev export CC_REPACKAGE_EXECUTOR=podman export CC_PODMAN_CC_DIR="$PWD/tests/fixtures/cc" export CC_PODMAN_OUT_DIR="$OUT_DIR" export CC_PODMAN_IMAGE="$IMAGE" export CC_PODMAN_SKIP_PIP=1 export CC_PODMAN_WARC_HTTP="http://host.containers.internal:${PORT}" echo "== 1. unit tests ==" $PY -m pytest -q 2>&1 | grep -vE "SyntaxWarning|re\.compile|URLRegex|handyurl|^\^\(" | tail -5 echo "== 2. build fixtures ==" $PY tests/make_fixtures.py "$CC_PODMAN_CC_DIR" 2>&1 | tail -1 echo "== 3. job image ==" if ! podman image exists "$IMAGE"; then podman build -f Dockerfile.job -t "$IMAGE" . else echo "$IMAGE present" fi echo "== 4. start range server ==" rm -rf "$OUT_DIR"; mkdir -p "$OUT_DIR" $PY scripts/range_server.py "$CC_PODMAN_CC_DIR" "$PORT" >/tmp/range_server.log 2>&1 & SRV=$! trap 'kill $SRV 2>/dev/null || true' EXIT sleep 1 echo "== 5. drive app (estimate + launch) via podman ==" $PY scripts/drive_app.py echo "== 6. validate output WARC ==" $PY - <<'PYEOF' import glob, os from warcio.archiveiterator import ArchiveIterator out = os.environ["CC_PODMAN_OUT_DIR"] files = sorted(glob.glob(os.path.join(out, "**", "repackaged*.warc.gz"), recursive=True)) assert files, f"no output WARC under {out}" total = 0 for f in files: types = [] with open(f, "rb") as fh: for rec in ArchiveIterator(fh): types.append(rec.rec_type) resp = types.count("response") total += resp print(f" {f}: {types.count('warcinfo')} warcinfo + {resp} response") assert total == 2, f"expected 2 response records, got {total}" print("OK: output WARC valid (2 response records)") PYEOF echo "ALL CHECKS PASSED"