code/finish_chain.sh

#!/usr/bin/env bash
# After the main crawl: (1) retry leftover repos (oversized tarballs -> git path only), (2) targeted repo_cap fill.
set -uo pipefail
cd /root/skills-db
P=.venv/bin/python
CRAWL_WORKERS=${CRAWL_WORKERS:-4}
$P code/github_bundles.py data/seeds/backfill_gitskills.txt data/raw/github_backfill "$CRAWL_WORKERS"
$P code/github_bundles.py data/seeds/pass2.txt data/raw/github_delta_p2 "$CRAWL_WORKERS"
$P code/ship_delta.py data/raw/github_delta_p2 pass2_new --once
$P code/github_bundles.py data/seeds/sg_new.txt data/raw/github_delta "$CRAWL_WORKERS"
$P code/ship_delta.py data/raw/github_delta pass1_sourcegraph --once
# wait until the backfill shipper has shipped everything the retry produced
while ls data/raw/github_backfill/files-*.parquet >/dev/null 2>&1; do sleep 60; done
$P code/fill_repo_cap.py || exit 1   # fail the job (so the watchdog retries) if the fill fails
echo "finish chain done"