code/crawl_chain.sh

#!/usr/bin/env bash
# Sequential GitHub crawl passes (one crawler at a time = bounded RAM). Each pass is resumable.
#  1. pass1_sourcegraph: popular repos with SKILL.md not in GitSkills
#  2. pass2_new: repos created since July (topic search) + sitemap repos     (waits for topic search to finish)
#  3. backfill_gitskills: GitSkills repos with multi-file skills, re-fetched so their bundles are complete
# Passes 1-2 are finished; re-running them only retries leftovers (oversized repos that need the git path),
# and any shards they produce are shipped right away (--once) because their ship services are retired.
set -uo pipefail
cd /root/skills-db
P=.venv/bin/python
W=${CRAWL_WORKERS:-10}
$P code/github_bundles.py data/seeds/sg_new.txt data/raw/github_delta "$W"
$P code/ship_delta.py data/raw/github_delta pass1_sourcegraph --once
while systemctl is-active --quiet skills-topics; do sleep 60; done
[ -s data/seeds/pass2.txt ] || $P code/build_pass2_seeds.py
$P code/github_bundles.py data/seeds/pass2.txt data/raw/github_delta_p2 "$W"
$P code/ship_delta.py data/raw/github_delta_p2 pass2_new --once
$P code/github_bundles.py data/seeds/backfill_gitskills.txt data/raw/github_backfill "$W"
echo "chain done"