scripts/verify_real.sh

#!/usr/bin/env bash
# Real-path verification against the LIVE commoncrawl HF bucket (needs HF_TOKEN).
# Runs the actual estimate code (CDN index read) + a real hf-cdn WARC fetch for a
# small domain/crawl, writing the output WARC locally, and validates it with warcio.
#
# Usage: HF_TOKEN=... scripts/verify_real.sh [DOMAIN] [CRAWL]
set -euo pipefail
cd "$(dirname "$0")/.."

PY=.venv/bin/python
CDXT=.venv/bin/cdxt
DOMAIN="${1:-commoncrawl.org}"
CRAWL="${2:-CC-MAIN-2026-21}"
RANGES=/tmp/cc-real-ranges.csv
OUT=/tmp/cc-real-out

: "${HF_TOKEN:?set HF_TOKEN (or run via an env that has it) to query the bucket}"
rm -rf "$OUT"; mkdir -p "$OUT"

echo "== estimate: CDN index scan for $DOMAIN @ $CRAWL =="
CC_INDEX_MODE=cdn CC_CRAWLS="$CRAWL" CC_DOMAINS="$DOMAIN" CC_HOSTNAMES="" \
  CC_RANGES_OUT="$RANGES" $PY -m src.index_query

N=$(($(wc -l < "$RANGES") - 1))
# The fetch step runs the real `cdxt repackage` CLI (as the HF fetch job does). cdxt
# is not a standing dev dependency — the estimate above uses the vendored SQL — so
# install it on demand here if it is missing, mirroring the job's runtime install.
if [ ! -x "$CDXT" ]; then
  echo "== installing cdx_toolkit for the fetch step (not a standing dev dep) =="
  uv pip install --python .venv \
    "cdx_toolkit[hf] @ git+https://github.com/commoncrawl/cdx_toolkit.git@feat/warc-range-sources" uvloop
fi
echo "== fetch: $N records from the commoncrawl bucket via hf-cdn =="
CDXT_UVLOOP=1 $CDXT -v repackage --target-source csv --csv-path "$RANGES" \
  --warc-download-prefix hf://buckets/commoncrawl/commoncrawl --hf-reader cdn \
  --prefix "$OUT/out" --processes 1 --parallel_readers 16 --size 1000000000 \
  2>&1 | grep -E "records extracted|execution time" || true

echo "== validate output WARC =="
$PY - <<PYEOF
import glob
from warcio.archiveiterator import ArchiveIterator
files = sorted(glob.glob("$OUT/out*.warc.gz"))
assert files, "no output WARC"
total = sum(1 for f in files for r in ArchiveIterator(open(f, "rb")) if r.rec_type == "response")
print(f"output files: {len(files)}, response records: {total}")
assert total == $N, f"expected $N records, got {total}"
print("OK: real CDN estimate + fetch validated")
PYEOF