File size: 5,012 Bytes
11ecc5b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 | #!/usr/bin/env bash
# Run the corpus pipeline after the download completes.
# Portable across jupyter-pod and kls-headnode.
#
# bash scripts/next.sh # check prerequisites only, run nothing
# bash scripts/next.sh --run # run schema -> corpus -> fertility
set -uo pipefail
REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
ANURAG="$(dirname "$REPO")"
ROOT="${VOICERAG_ROOT:-$ANURAG/voicerag_data}"
RUN=0
[ "${1:-}" = "--run" ] && RUN=1
cd "$REPO"
mkdir -p logs
echo "════════ CONTEXT ════════"
echo " host : $(hostname -s)"
echo " repo : $REPO"
echo " root : $ROOT"
# ---------------------------------------------------------------- 1. data
echo
echo "════════ 1. DATASET ════════"
HUB="$ROOT/hf_cache/hub"
if [ -d "$HUB" ]; then
echo " hf_cache: $(du -sh "$ROOT/hf_cache" 2>/dev/null | cut -f1)"
for r in datasets--ai4bharat--MSMARCO-XI models--BAAI--bge-m3 \
models--BAAI--bge-reranker-v2-m3 models--sentence-transformers--all-MiniLM-L6-v2; do
[ -d "$HUB/$r" ] && echo " ok $r" || echo " MISSING $r"
done
NPQ=$(find "$HUB" -name '*.parquet' 2>/dev/null | wc -l)
echo " parquet files: $NPQ"
# HF caches use RELATIVE symlinks (snapshots -> ../../blobs), so a move is safe.
# Verify one actually resolves after any relocation.
SAMPLE=$(find "$HUB" -name '*.parquet' 2>/dev/null | head -1)
if [ -n "$SAMPLE" ]; then
if [ -r "$SAMPLE" ] && [ -s "$SAMPLE" ]; then
echo " symlinks resolve: ok ($(du -Lh "$SAMPLE" 2>/dev/null | cut -f1) sample)"
else
echo " !! symlinks BROKEN after move — re-run 01_download.py to repair"
exit 1
fi
fi
else
echo " !! no hf_cache at $ROOT"
echo " If the download ran elsewhere: bash scripts/relocate.sh"
exit 1
fi
# ---------------------------------------------------------------- 2. python
echo
echo "════════ 2. PYTHON DEPENDENCIES ════════"
MISSING=$(python3 - <<'PY'
need = {"polars":"schema+corpus", "pyarrow":"schema", "numpy":"all",
"transformers":"fertility", "matplotlib":"fertility plot"}
missing = []
for m, why in need.items():
try:
mod = __import__(m)
print(f" ok {m:14s} {getattr(mod,'__version__','?'):12s} ({why})")
except Exception:
print(f" MISSING {m:14s} {'':12s} ({why})")
missing.append(m)
print("MISSING:" + ",".join(missing))
PY
)
echo "$MISSING" | grep -v '^MISSING:'
MISS=$(echo "$MISSING" | sed -n 's/^MISSING://p')
if [ -n "$MISS" ]; then
echo
echo " !! Missing: $MISS"
echo " These exist on jupyter-pod. Either run this there, or create an"
echo " isolated venv here (nothing system-wide is touched):"
echo
echo " python3 -m venv --system-site-packages $ANURAG/.venv"
echo " source $ANURAG/.venv/bin/activate"
echo " pip install ${MISS//,/ }"
echo
[ "$RUN" -eq 1 ] && { echo " aborting."; exit 1; }
fi
# ---------------------------------------------------------------- 3. status
echo
echo "════════ 3. PIPELINE STATUS ════════"
step() { [ -e "$2" ] && echo " done $1 ($(du -sh "$2" 2>/dev/null | cut -f1))" \
|| echo " pending $1"; }
step "schema_report.json " "$ROOT/data/schema_report.json"
step "passages.parquet " "$ROOT/data/passages.parquet"
step "pseudo_docs.parquet" "$ROOT/data/pseudo_docs.parquet"
step "fertility.json " "$ROOT/results/fertility.json"
if [ "$RUN" -eq 0 ]; then
cat <<EOF
════════ TO RUN ════════
export VOICERAG_ROOT="$ROOT"
bash scripts/next.sh --run
Or step by step (each writes only under \$VOICERAG_ROOT):
python3 scripts/02_inspect_schema.py --root "\$VOICERAG_ROOT"
python3 src/pseudo_docs.py --root "\$VOICERAG_ROOT" --strategy url
python3 src/fertility.py --root "\$VOICERAG_ROOT"
EOF
exit 0
fi
export VOICERAG_ROOT="$ROOT"
# ---------------------------------------------------------------- run
echo
echo "════════ RUNNING ════════"
echo
echo "-- step 1/3: schema inspection (fast) --"
python3 scripts/02_inspect_schema.py --root "$ROOT" 2>&1 | tee logs/schema.log | tail -45
[ -f "$ROOT/data/schema_report.json" ] || { echo " !! schema report not written; see logs/schema.log"; exit 1; }
echo
echo "-- step 2/3: fertility (needs only the schema report; ~5 min) --"
python3 src/fertility.py --root "$ROOT" 2>&1 | tee logs/fertility.log | tail -30
echo
echo "-- step 3/3: corpus build (memory-heavy; backgrounded) --"
nohup python3 src/pseudo_docs.py --root "$ROOT" --strategy url --target-words 3000 \
> logs/corpus.log 2>&1 &
echo " started pid $! -> logs/corpus.log"
echo " watch: tail -f $REPO/logs/corpus.log"
cat <<EOF
════════ SEND ME ════════
$ROOT/data/schema_report.json
$ROOT/results/fertility.json
$REPO/logs/corpus.log (once it finishes)
EOF
|