File size: 2,295 Bytes
22741d9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 | #!/usr/bin/env bash
# Truly detached Feather training launcher — survives Hermes session transitions.
# Writes PID to ~/.cache/autoresearch/train_pid and logs to run_3060_detached.log.
set -euo pipefail
REPO="/home/mikeb/work/feather"
cd "$REPO"
# Kill any stale training
pkill -9 -f "python.*train\.py" 2>/dev/null || true
sleep 1
HF_TOKEN_VAL=$(grep -ohP 'hf_[A-Za-z0-9_-]+' ~/.bashrc 2>/dev/null | head -1 || true)
# Truly detach: setsid + nohup + close all fds
exec setsid /usr/bin/env \
LD_LIBRARY_PATH=/usr/lib/wsl/lib:/usr/local/cuda/lib64 \
PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
HF_TOKEN="$HF_TOKEN_VAL" \
HUGGINGFACE_HUB_TOKEN="$HF_TOKEN_VAL" \
WANDB_DISABLED=true \
HYDRA_USE_NEMOTRON=1 \
HYDRA_USE_FULL_BLEND=1 \
HYDRA_SAMPLED_SOFTMAX=512 \
HYDRA_SOFTCAP_CLAMP=1 \
HYDRA_SEQ_LEN=1024 \
HYDRA_HEADDIM=32 \
HYDRA_D_STATE=64 \
HYDRA_TIME_BUDGET=43200 \
HYDRA_ENGRAM_TOPK=64 \
HYDRA_CANTOR_DISABLE=0 \
HYDRA_CANTOR_LEARNABLE=1 \
HYDRA_CANTOR_SCORE_GRAD=1 \
HYDRA_ENGRAM_ROUTING=auto \
HYDRA_REALITY_BRIDGE=1 \
HYDRA_SEMANTIC_SMOOTH_STD=0.01 \
HYDRA_SLOW_FAST_ORTHO_METRICS=1 \
HYDRA_SLOW_FAST_ORTHO_LAMBDA=1e-4 \
HYDRA_GDN_LAYERS= \
HYDRA_MTP_K=1 \
HYDRA_USE_MDLM=0 \
HYDRA_MUON_COMPILE=0 \
HYDRA_MUON_NS_STEPS=2 \
HYDRA_MATRIX_LR=0.10 \
HYDRA_EMBED_LR=1.3 \
HYDRA_UNEMBED_LR=0.004 \
HYDRA_DT_BIAS_LR=0.15 \
HYDRA_SCALAR_LR=0.05 \
HYDRA_WARMUP_RATIO=0.01 \
HYDRA_LR_MIN_MULT=0.10 \
HYDRA_DOC_SEP_MASK=1 \
HYDRA_STREAM_SHUFFLE_BUFFER=4096 \
HYDRA_LOCAL_SHARDS_ONLY=0 \
HYDRA_BACKGROUND_PREFETCH=0 \
HYDRA_STREAM_PREFETCH=16 \
HYDRA_TOKEN_PREFETCH=4 \
HYDRA_TOKEN_CACHE_GB=1 \
HYDRA_CKPT_INTERVAL=500 \
HYDRA_MID_VAL_INTERVAL=500 \
HYDRA_EVAL_BATCH=1 \
HYDRA_EVAL_TOKENS=51200 \
HYDRA_CE_CHUNK=32 \
HYDRA_SKIP_FACTUAL_EVAL=1 \
HYDRA_RESUME_CKPT=/home/mikeb/.cache/autoresearch/latest.pt \
HYDRA_N_LAYER=6 \
HYDRA_D_MODEL=192 \
HYDRA_EXPAND=3 \
HYDRA_BATCH_SIZE=16 \
HYDRA_TOTAL_BATCH=32768 \
HYDRA_HYENA_LAYERS= \
HYDRA_HTM_SUBSAMPLE=16 \
UV_PYTHON=/usr/bin/python3 \
taskset -c 0-15 /home/mikeb/work/feather/.venv/bin/python -u train.py \
</dev/null >/home/mikeb/work/feather/run_3060_detached.log 2>&1 &
TPID=$!
echo "$TPID" > /home/mikeb/.cache/autoresearch/train_pid
echo "Launched PID $TPID — fully detached from Hermes session"
disown "$TPID" 2>/dev/null || true
|