File size: 2,396 Bytes
ddf8c5b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 | #!/usr/bin/env bash
# k3-test/run_all.sh — full runbook driver (plan §4 Phases 0–3).
# Usage:
# tmux new -s k3
# ./run_all.sh # everything, K3 download starts in background immediately
# SKIP_DOWNLOAD=1 ./run_all.sh # if K3 already downloaded
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/lib/common.sh"
trap 'stop_monitors; unfill_ram' EXIT
info "k3-test run starting. Logs: $LOG_ROOT"
# ---- Phase 0: setup + kick the big download OFF in the background ------------
"$SCRIPT_DIR/setup.sh" 2>&1 | tee "$LOG_ROOT/setup.log"
if [[ -z "${SKIP_DOWNLOAD:-}" ]]; then
info "launching K3 download in background ($K3_INCLUDE, ~1.5 TB)…"
nohup "$SCRIPT_DIR/01_download_k3.sh" > "$LOG_ROOT/download_k3.log" 2>&1 &
DL_PID=$!
echo "$DL_PID" > "$LOG_ROOT/download_k3.pid"
fi
"$SCRIPT_DIR/02_download_smoke.sh" 2>&1 | tee -a "$LOG_ROOT/download_smoke.log"
# ---- Phase 0b: recon (free, while downloading) --------------------------------
"$SCRIPT_DIR/00_recon.sh" 2>&1 | tee "$LOG_ROOT/recon_driver.log"
# ---- Phase 1: smoke gate -------------------------------------------------------
if ! "$SCRIPT_DIR/10_smoke.sh" 2>&1 | tee "$LOG_ROOT/smoke_driver.log"; then
die "SMOKE GATE FAILED — investigate logs in $LOG_ROOT before spending more"
fi
# ---- gate: wait for K3 ----------------------------------------------------------
if [[ -n "${DL_PID:-}" ]]; then
info "waiting for K3 download (pid $DL_PID) — tail -f $LOG_ROOT/download_k3.log to watch"
while kill -0 "$DL_PID" 2>/dev/null; do
du -sh "${MODEL_DIR:-$HOME/models}/Kimi-K3-GGUF" 2>/dev/null | tr '\n' ' '
df -h . | tail -1 | awk '{print " free:",$4}'
sleep 120
done
wait "$DL_PID" || die "K3 download FAILED — check $LOG_ROOT/download_k3.log"
fi
# ---- Phase 2: experiments -------------------------------------------------------
"$SCRIPT_DIR/20_k3_experiments.sh" 2>&1 | tee "$LOG_ROOT/experiments_driver.log"
# ---- Phase 3: pack --------------------------------------------------------------
"$SCRIPT_DIR/30_teardown.sh"
cat <<EOF
================================================================================
RUN COMPLETE. Sync $LOG_ROOT and the tarball off-box, then DESTROY the instance.
Report against k3-rental-test-plan.md §6 success criteria.
================================================================================
EOF
|