scripts/verify_local.sh

#!/usr/bin/env bash
# End-to-end local verification with podman (no HF/AWS, no cost).
#   1. unit tests
#   2. (re)build toy fixtures
#   3. build the job image (cdx_toolkit preinstalled) if missing
#   4. start a Range-capable HTTP server for the fixture WARCs
#   5. drive the app's Estimate + Launch buttons through the real podman runner
#   6. validate the produced WARC with warcio
set -euo pipefail
cd "$(dirname "$0")/.."

PY=.venv/bin/python
PORT=8080
OUT_DIR="$PWD/.verify-out"
IMAGE=cc-repackage-job:dev

export CC_REPACKAGE_EXECUTOR=podman
export CC_PODMAN_CC_DIR="$PWD/tests/fixtures/cc"
export CC_PODMAN_OUT_DIR="$OUT_DIR"
export CC_PODMAN_IMAGE="$IMAGE"
export CC_PODMAN_SKIP_PIP=1
export CC_PODMAN_WARC_HTTP="http://host.containers.internal:${PORT}"

echo "== 1. unit tests =="
$PY -m pytest -q 2>&1 | grep -vE "SyntaxWarning|re\.compile|URLRegex|handyurl|^\^\(" | tail -5

echo "== 2. build fixtures =="
$PY tests/make_fixtures.py "$CC_PODMAN_CC_DIR" 2>&1 | tail -1

echo "== 3. job image =="
if ! podman image exists "$IMAGE"; then
  podman build -f Dockerfile.job -t "$IMAGE" .
else
  echo "$IMAGE present"
fi

echo "== 4. start range server =="
rm -rf "$OUT_DIR"; mkdir -p "$OUT_DIR"
$PY scripts/range_server.py "$CC_PODMAN_CC_DIR" "$PORT" >/tmp/range_server.log 2>&1 &
SRV=$!
trap 'kill $SRV 2>/dev/null || true' EXIT
sleep 1

echo "== 5. drive app (estimate + launch) via podman =="
$PY scripts/drive_app.py

echo "== 6. validate output WARC =="
$PY - <<'PYEOF'
import glob, os
from warcio.archiveiterator import ArchiveIterator
out = os.environ["CC_PODMAN_OUT_DIR"]
files = sorted(glob.glob(os.path.join(out, "**", "repackaged*.warc.gz"), recursive=True))
assert files, f"no output WARC under {out}"
total = 0
for f in files:
    types = []
    with open(f, "rb") as fh:
        for rec in ArchiveIterator(fh):
            types.append(rec.rec_type)
    resp = types.count("response")
    total += resp
    print(f"  {f}: {types.count('warcinfo')} warcinfo + {resp} response")
assert total == 2, f"expected 2 response records, got {total}"
print("OK: output WARC valid (2 response records)")
PYEOF

echo "ALL CHECKS PASSED"