File size: 2,644 Bytes
3d7c238 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 | #!/usr/bin/env bash
# Copy code, the export and the robustness inputs to the pod, or bring runs back (run on this machine).
# sync.sh up HOST PORT classifier/ -> /workspace/code/classifier, export -> /workspace/data, inputs
# sync.sh down HOST PORT /workspace/runs and the check files -> ~/twitter_models/classifier-v1/
# Uses the pod's direct "SSH over exposed TCP" address (the ssh.runpod.io proxy does not support rsync) and only
# the RunPod key (SSH_KEY, default ~/.ssh/dom_runpod) with IdentitiesOnly, so other keys are never offered.
# Uploads use -rl --checksum, not -a or -t: the RunPod network volume (FUSE) refuses chown and setting file times,
# so changes are found by checksum. No credentials or audit logs are shipped.
set -euo pipefail
HERE=$(cd "$(dirname "$0")/.." && pwd) # .../evaluation/market_psych_v2/classifier
REPO=$(cd "$HERE/../../.." && pwd)
dir=${1:-}; host=${2:?host}; port=${3:?port}
KEY=${SSH_KEY:-$HOME/.ssh/dom_runpod}
SSH="ssh -p $port -i $KEY -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new"
DATA=$REPO/data/market_psych/development/classifier-data-20261017
INPUTS=$REPO/data/market_psych/development/classifier-v1-20261017/inputs
LOCAL=${LOCAL_RUNS:-$HOME/twitter_models/classifier-v1}
case "$dir" in
up)
# The hash covers exactly what rsync ships (tests/ and caches excluded); untracked or modified files mark the
# commit dirty, so the stamp never claims a clean commit for code that is not in HEAD.
tree=$(cd "$HERE" && find . -type f ! -path './tests/*' ! -path '*/__pycache__/*' ! -name '.source_commit' \
! -name '*.pyc' -print0 | sort -z | xargs -0 sha256sum | sha256sum | cut -c1-16)
commit=$(git -C "$REPO" rev-parse HEAD)
[ -z "$(git -C "$REPO" status --porcelain -- "$HERE")" ] || commit="$commit-dirty"
echo "$commit tree:$tree" > "$HERE/.source_commit"
$SSH "root@$host" "mkdir -p /workspace/code /workspace/data /workspace/inputs /workspace/runs"
rsync -rlz --checksum --delete --exclude __pycache__ --exclude tests -e "$SSH" "$HERE/" "root@$host:/workspace/code/classifier/"
rsync -rlz --checksum -e "$SSH" "$DATA/" "root@$host:/workspace/data/classifier-data-20261017/"
rsync -rlz --checksum -e "$SSH" "$INPUTS/" "root@$host:/workspace/inputs/"
echo "shipped $(cat "$HERE/.source_commit")" ;;
down)
mkdir -p "$LOCAL"
rsync -rltz -e "$SSH" --exclude 'last/trainer.pt' "root@$host:/workspace/runs/" "$LOCAL/runs/"
for f in check_env.json selftest.json nvidia-smi.txt; do
rsync -rltz -e "$SSH" "root@$host:/workspace/$f" "$LOCAL/" || true
done ;;
*) sed -n 2,7p "$0"; exit 2 ;;
esac
|