init

.gitignore

@@ -0,0 +1,4 @@
+.hf_token
+__pycache__/
+*.pyc
+validate/*.tmp

README.md

@@ -0,0 +1,27 @@
+# SmallThinker-4B-A0.6B — 熱參數在 RAM、冷參數在 SSD
+
+用 llama.cpp 跑 `Tiiny/SmallThinker-4BA0.6B-Instruct-GGUF` 的
+`SmallThinker-4B-A0.6B-Instruct.Q4_K.gguf`（2.45 GiB，MoE：32 層 × 32 experts，
+每 token 用 4 個）。權重永遠以 file-backed `mmap` 留在 SSD，RAM 只放熱的權重
+（expert arena）＋ KV ＋ compute buffer。
+
+```bash
+./llama_server.sh              # port 8080
+./llama_server.sh --plan       # 只印推導出來的參數
+./llama_server.sh --verify     # 啟動 + 打一次 chat + 量記憶體
+ST_RAM_BUDGET_MB=512 ./llama_server.sh   # 手動壓低 RAM 預算做實驗
+```
+
+進度真相來源：[STATUS.md](STATUS.md)。續作指引：[AGENTS.md](AGENTS.md)。
+
+## 檔案
+
+| 檔案 | 用途 |
+| --- | --- |
+| `llama_server.sh` | 一鍵啟動（自動抓 llama.cpp / 套 patch / 編譯 / 下載權重） |
+| `patches/` | llama.cpp 改動（expert 分頁） |
+| `tools/verify_run.py` | 打一次 chat request 並量 total RSS / swap / SSD 讀取 |
+| `tools/io_probe.py` | 實測本機 SSD 讀取能力（**換機器必跑**） |
+| `tools/drop_model_cache.py` | 量測前丟掉模型檔的 page cache |
+| `validate/` | 驗證證據（JSON） |
+| `sync.sh` | 推回 HF（崩潰後的唯一保險） |

llama_server.sh

@@ -0,0 +1,456 @@
+#!/usr/bin/env bash
+# ==============================================================================
+# SmallThinker-4B-A0.6B-Instruct — 熱參數在 RAM、冷參數在 SSD 的啟動器
+#
+#   ./llama_server.sh              # port 8080
+#   ./llama_server.sh --port 9000  # 換 port
+#   ./llama_server.sh --plan       # 只印推導出來的參數，不動任何東西
+#   ./llama_server.sh --verify     # 啟動後打一次 /v1/chat/completions 並量記憶體
+#
+# 這個檔案假設「環境還沒建立」：什麼都沒有時，它會自己抓 llama.cpp、套 patch、
+# 編譯、把權重抓回來，最後啟動 llama-server。已經建立好的話就跳過對應步驟。
+#
+# 三個必須先講清楚的前提（不然這個設計看起來很怪）：
+#
+# 1. **模型與量化不變。** 權重永遠是 Tiiny/SmallThinker-4BA0.6B-Instruct-GGUF 的
+#    SmallThinker-4B-A0.6B-Instruct.Q4_K.gguf，2630212704 bytes（2.45 GiB），
+#    永遠只做 SSD 上的 file-backed mmap。RAM 少的機器**不會**換一個小一點的 GGUF。
+#
+# 2. **跟著機器變的是「RAM 放得下多少權重」。** RAM 越大 → arena 越大 → 命中率越高
+#    → SSD 讀取越少。分頁器自己會算，這裡只需要把 RAM 預算餵對（ST_RAM_BUDGET_MB）。
+#
+# 3. **不打開 swap。** 匿名記憶體超出預算就是 crash，而不是安靜地 swap 掉。
+# ==============================================================================
+set -uo pipefail
+
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
+
+# ------------------------------------------------------------------ 設定
+# 這些是「這個模型的事實」，不是可以調的參數；要改改這裡，不要在呼叫時傳參數。
+HF_REPO="${ST_HF_REPO:-HelloSun/SmallThinker4b}"
+LLAMA_REPO="${ST_LLAMA_REPO:-https://github.com/ggml-org/llama.cpp.git}"
+# patch 是針對這個 commit 生成的。換 commit 一定要重新產生 patch。
+LLAMA_COMMIT="${ST_LLAMA_COMMIT:-b9acf138a1e28ce1fc23b5a4fc4b12444b50f7ea}"
+
+MODEL_REPO="Tiiny/SmallThinker-4BA0.6B-Instruct-GGUF"
+MODEL_FILE="SmallThinker-4B-A0.6B-Instruct.Q4_K.gguf"
+MODEL_SIZE=2630212704          # bytes，驗證下載用
+
+MODEL_ARCH="smallthinker"
+N_LAYER=32                     # smallthinker.block_count
+N_EMBD=1536                    # smallthinker.embedding_length
+N_EXPERT=32                    # smallthinker.expert_count
+N_EXPERT_USED=4                # smallthinker.expert_used_count
+EXPERT_FFN=768                 # smallthinker.expert_feed_forward_length
+N_HEAD=12                      # smallthinker.attention.head_count
+N_HEAD_KV=2                    # smallthinker.attention.head_count_kv
+KEY_LEN=128                    # smallthinker.attention.key_length
+VAL_LEN=128                    # smallthinker.attention.value_length
+MODEL_MAX_CTX=32768            # smallthinker.context_length
+
+# 單一 expert 權重 = gate + up + down 三段，元素數 3 * N_EMBD * EXPERT_FFN。
+# Q4_K 每元素實際約 0.5625 B/elem（GGUF metadata 實測），這裡保守用 0.6。
+EXPERT_ELEMS=$(( 3 * N_EMBD * EXPERT_FFN ))
+EXPERT_BYTES=$(( EXPERT_ELEMS * 6 / 10 ))
+EXPERT_TOTAL_MIB=$(( N_LAYER * N_EXPERT * EXPERT_BYTES / 1048576 ))
+
+# 工作目錄：原始碼、patch、模型、執行檔都放這裡。
+WORK_DIR="${ST_HOME:-${XDG_CACHE_HOME:-$HOME/.cache}/smallthinker4b}"
+SRC_DIR="$WORK_DIR/llama.cpp"
+BIN_DIR="$WORK_DIR/bin"
+MODEL_DIR="${ST_MODEL_DIR:-$WORK_DIR/models}"
+PATCH_DIR="$SCRIPT_DIR/patches"
+LOG_DIR="$WORK_DIR/logs"
+
+# ------------------------------------------------------------------ 參數
+PORT=8080
+PLAN_ONLY=0
+VERIFY=0
+while [ $# -gt 0 ]; do
+    case "$1" in
+        --port)   PORT="${2:?--port 需要一個數字}"; shift ;;
+        --plan)   PLAN_ONLY=1 ;;
+        --verify) VERIFY=1 ;;
+        -h|--help) sed -n '2,20p' "$0"; exit 0 ;;
+        *) die() { printf '[st4b] 未知參數：%s（只有 --port / --plan / --verify）\n' "$1" >&2; exit 2; }; die "$1" ;;
+    esac
+    shift
+done
+
+say()  { printf '[st4b] %s\n' "$*" >&2; }
+step() { printf '\033[1;34m==>\033[0m %s\n' "$*" >&2; }
+die()  { printf '[st4b] ERROR: %s\n' "$*" >&2; exit 1; }
+
+command -v git    >/dev/null || die "需要 git"
+command -v cmake  >/dev/null || die "需要 cmake（apt install cmake build-essential）"
+command -v curl   >/dev/null || die "需要 curl"
+command -v python3 >/dev/null || die "需要 python3"
+
+# ======================================================== 1. 偵測系統資源
+# 放在最前面：後面所有步驟（尤其是 RAM 預算）都要用到。
+MIB=1048576
+
+meminfo_kb() {
+    awk -v k="$1:" '$1==k {print $2; exit} END{}' /proc/meminfo 2>/dev/null || echo 0
+}
+
+cgroup_mem_limit() {
+    local v
+    if [ -r /sys/fs/cgroup/memory.max ]; then
+        v=$(tr -d '[:space:]' </sys/fs/cgroup/memory.max)
+        case "$v" in ''|max) ;; *) printf '%s' "$v"; return 0 ;; esac
+    fi
+    if [ -r /sys/fs/cgroup/memory/memory.limit_in_bytes ]; then
+        v=$(tr -d '[:space:]' </sys/fs/cgroup/memory/memory.limit_in_bytes)
+        if [ -n "$v" ] && [ "$v" -lt 4611686018427387904 ] 2>/dev/null; then
+            printf '%s' "$v"; return 0
+        fi
+    fi
+    return 1
+}
+
+cgroup_mem_used() {
+    if   [ -r /sys/fs/cgroup/memory.current ];            then tr -d '[:space:]' </sys/fs/cgroup/memory.current
+    elif [ -r /sys/fs/cgroup/memory/memory.usage_in_bytes ]; then tr -d '[:space:]' </sys/fs/cgroup/memory/memory.usage_in_bytes
+    else echo 0
+    fi
+}
+
+cgroup_cpu_quota() {
+    local q p
+    if [ -r /sys/fs/cgroup/cpu.max ]; then
+        read -r q p < /sys/fs/cgroup/cpu.max
+        if [ "$q" != "max" ] && [ -n "${p:-}" ] && [ "$p" -gt 0 ] 2>/dev/null; then
+            echo $(( q / p )); return 0
+        fi
+        return 1
+    fi
+    return 1
+}
+
+affinity_cpus() { nproc 2>/dev/null || getconf _NPROCESSORS_ONLN 2>/dev/null || echo 1; }
+
+gpu_mb() {
+    command -v nvidia-smi >/dev/null 2>&1 || { echo 0; return; }
+    nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits 2>/dev/null \
+        | awk '{s+=$1} END{print s+0}'
+}
+
+# --- RAM -------------------------------------------------------------------
+# 容器裡 /proc/meminfo 的 MemAvailable 是**宿主**的餘量，直接用會高估，
+# 必須跟 cgroup 上限取較小值。這個坑在 sddqwen35a3b_v01 踩過一次。
+AVAIL_KB=$(meminfo_kb MemAvailable)
+[ "${AVAIL_KB:-0}" -gt 0 ] || AVAIL_KB=$(meminfo_kb MemFree)
+AVAIL_BYTES=$(( ${AVAIL_KB:-0} * 1024 ))
+if LIMIT=$(cgroup_mem_limit); then
+    FREE=$(( LIMIT - $(cgroup_mem_used) ))
+    [ "$FREE" -lt 0 ] 2>/dev/null && FREE=0
+    AVAIL_BYTES=$(( AVAIL_BYTES < FREE ? AVAIL_BYTES : FREE ))
+fi
+[ "$AVAIL_BYTES" -gt 0 ] || die "讀不到可用的 RAM"
+USABLE_MB=$(( AVAIL_BYTES / MIB ))
+CGROUP_MB=$( { cgroup_mem_limit || echo 0; } | awk -v m=$MIB '{printf "%d", $1/m}')
+
+# --- CPU -------------------------------------------------------------------
+CPUS=$(affinity_cpus)
+QUOTA=$(cgroup_cpu_quota || true)
+[ -n "${QUOTA:-}" ] && [ "$QUOTA" -lt "$CPUS" ] 2>/dev/null && CPUS="$QUOTA"
+# SmallThinker 每層每 token 只有 n_expert_used=4 個 expert 可以平行，
+# threads 超過 4 之後只是多開執行緒搶同一批 CPU。
+THREADS=$(( CPUS > 8 ? 8 : CPUS ))
+[ "$THREADS" -lt 1 ] && THREADS=1
+
+GPU_MB=$(gpu_mb)
+
+# --- 推導啟動參數 -----------------------------------------------------------
+# ST_RAM_BUDGET_MB 可以手動壓低做實驗（冷權重真的留在 SSD）；沒給就自動偵測。
+CTX=${ST_CTX:-4096}
+UBATCH=${ST_UBATCH:-512}
+# KV：每個 token = N_LAYER * N_HEAD_KV * (KEY_LEN + VAL_LEN) 個元素，f16 = 2 bytes。
+KV_ELEMS=$(( N_LAYER * N_HEAD_KV * (KEY_LEN + VAL_LEN) ))
+KV_ESTIMATE_MB=$(( CTX * KV_ELEMS * 2 / MIB ))
+KV_RESERVE_MB=${ST_KV_RESERVE_MB:-$KV_ESTIMATE_MB}
+[ "$KV_RESERVE_MB" -lt "$KV_ESTIMATE_MB" ] && KV_RESERVE_MB="$KV_ESTIMATE_MB"
+COMPUTE_RESERVE_MB=${ST_COMPUTE_RESERVE_MB:-$(( UBATCH * 2 + 800 ))}
+RESERVE_MB=$(( KV_RESERVE_MB + COMPUTE_RESERVE_MB ))
+
+# 非 expert 的權重（embedding / attention / norms / router）必須常駐 RAM，
+# 否則每 token 都要重讀。粗估 = 模型總大小 − expert 總量。
+NON_EXPERT_MIB=$(( $(stat -c %s "$MODEL_DIR/$MODEL_FILE" 2>/dev/null || echo $MODEL_SIZE) / MIB - EXPERT_TOTAL_MIB ))
+[ "$NON_EXPERT_MIB" -lt 0 ] && NON_EXPERT_MIB=0
+
+if [ -n "${ST_RAM_BUDGET_MB:-}" ]; then
+    RAM_BUDGET_MB=$ST_RAM_BUDGET_MB
+    BUDGET_SRC="ST_RAM_BUDGET_MB（手動）"
+else
+    # 自動：可用 RAM 扣掉系統保留就是模型預算。
+    SYSTEM_RESERVE_MB=${ST_SYSTEM_RESERVE_MB:-2048}
+    RAM_BUDGET_MB=$(( USABLE_MB - SYSTEM_RESERVE_MB ))
+    [ "$RAM_BUDGET_MB" -gt 0 ] || die "可用 RAM ${USABLE_MB} MiB 不足系統保留 ${SYSTEM_RESERVE_MB} MiB"
+    BUDGET_SRC="自動偵測（可用 ${USABLE_MB} − 系統保留 ${SYSTEM_RESERVE_MB}）"
+fi
+
+# arena = 預算 − 非 expert 權重（常駐） − KV − compute
+ARENA_MB=$(( RAM_BUDGET_MB - NON_EXPERT_MIB - RESERVE_MB ))
+[ "$ARENA_MB" -lt 0 ] && ARENA_MB=0
+EXPERT_SLOTS=$(( ARENA_MB * MIB / EXPERT_BYTES ))
+
+say "RAM 可用 ${USABLE_MB} MiB（cgroup 上限 ${CGROUP_MB} MiB）"
+say "CPU ${THREADS} threads（配額 ${QUOTA:-無}／可用核心 $(affinity_cpus)）、GPU ${GPU_MB} MiB"
+say "模型結構：${N_LAYER} 層 × ${N_EXPERT} experts（每 token 用 ${N_EXPERT_USED}），單一 expert ≈ $(( EXPERT_BYTES / 1024 )) KiB"
+say "RAM 預算 ${RAM_BUDGET_MB} MiB（來源：${BUDGET_SRC}）"
+say "推導：非 expert 權重=${NON_EXPERT_MIB} MiB（常駐）  KV=${KV_RESERVE_MB} MiB  compute=${COMPUTE_RESERVE_MB} MiB"
+say "可配置 arena=${ARENA_MB} MiB（約 ${EXPERT_SLOTS} 個 expert 槽；expert 總量 ${EXPERT_TOTAL_MIB} MiB）"
+
+refresh_memory_budget() {
+    # Build/download 可能改變 page-cache/cgroup 用量，啟動前一定要重新讀一次。
+    local avail_kb avail_bytes limit free
+    avail_kb=$(meminfo_kb MemAvailable)
+    [ "${avail_kb:-0}" -gt 0 ] || avail_kb=$(meminfo_kb MemFree)
+    avail_bytes=$(( ${avail_kb:-0} * 1024 ))
+    if limit=$(cgroup_mem_limit); then
+        free=$(( limit - $(cgroup_mem_used) ))
+        [ "$free" -lt 0 ] 2>/dev/null && free=0
+        avail_bytes=$(( avail_bytes < free ? avail_bytes : free ))
+        CGROUP_MB=$(( limit / MIB ))
+    fi
+    [ "$avail_bytes" -gt 0 ] || die "啟動前讀不到可用 RAM"
+    USABLE_MB=$(( avail_bytes / MIB ))
+    if [ -z "${ST_RAM_BUDGET_MB:-}" ]; then
+        SYSTEM_RESERVE_MB=${ST_SYSTEM_RESERVE_MB:-2048}
+        RAM_BUDGET_MB=$(( USABLE_MB - SYSTEM_RESERVE_MB ))
+        [ "$RAM_BUDGET_MB" -gt 0 ] || die "啟動前可用 RAM ${USABLE_MB} MiB 不足系統保留 ${SYSTEM_RESERVE_MB} MiB"
+        BUDGET_SRC="自動偵測（可用 ${USABLE_MB} − 系統保留 ${SYSTEM_RESERVE_MB}）"
+    fi
+    ARENA_MB=$(( RAM_BUDGET_MB - NON_EXPERT_MIB - RESERVE_MB ))
+    [ "$ARENA_MB" -lt 0 ] && ARENA_MB=0
+    EXPERT_SLOTS=$(( ARENA_MB * MIB / EXPERT_BYTES ))
+    [ "$EXPERT_SLOTS" -lt 0 ] && EXPERT_SLOTS=0
+    say "啟動前重新偵測：可用 RAM=${USABLE_MB} MiB，預算=${RAM_BUDGET_MB} MiB，arena=${ARENA_MB} MiB"
+}
+
+# ============================================ 2. 取得 patch（自我安裝）
+hf_get() {   # hf_get <repo> <path> <dest>
+    local url="https://huggingface.co/$1/resolve/main/$2"
+    mkdir -p "$(dirname "$3")"
+    curl -fsSL -H "Authorization: Bearer ${HF_TOKEN:-}" -o "$3" "$url" \
+        || die "抓不到 $2（請確認 $HF_REPO 有這個檔案，或把 patches/ 放在腳本旁邊）"
+}
+
+ensure_patches() {
+    mkdir -p "$PATCH_DIR"
+    if [ -n "$(ls -A "$PATCH_DIR"/*.patch 2>/dev/null)" ]; then
+        say "使用本地的 patch：$(cd "$PATCH_DIR" && ls *.patch | tr '\n' ' ')"
+        return
+    fi
+    step "從 $HF_REPO 取得 patch"
+    local p
+    for p in 0001-st-expert-pager.patch; do
+        hf_get "$HF_REPO" "patches/$p" "$PATCH_DIR/$p" && say "  patches/$p"
+    done
+    [ -n "$(ls -A "$PATCH_DIR"/*.patch 2>/dev/null)" ] || say "  （這個 repo 目前沒有 patch，用上游行為）"
+}
+
+# ================================================== 3. 取得 llama.cpp 原始碼
+ensure_source() {
+    if [ -d "$SRC_DIR/.git" ]; then
+        local have
+        have=$(git -C "$SRC_DIR" rev-parse HEAD 2>/dev/null || echo none)
+        if [ "$have" = "$LLAMA_COMMIT" ] && [ -f "$SRC_DIR/.st-patched" -o -f "$SRC_DIR/.st-unpatched" ]; then
+            say "llama.cpp 已經在 $LLAMA_COMMIT 且已處理 patch"
+            return
+        fi
+    fi
+    step "取得 llama.cpp $LLAMA_COMMIT"
+    mkdir -p "$WORK_DIR"
+    if [ ! -d "$SRC_DIR/.git" ]; then
+        rm -rf "$SRC_DIR"
+        git clone -q --filter=blob:none "$LLAMA_REPO" "$SRC_DIR" \
+            || die "git clone 失敗（檢查網路）"
+    fi
+    git -C "$SRC_DIR" fetch -q --depth 1 origin "$LLAMA_COMMIT" 2>/dev/null || true
+    git -C "$SRC_DIR" checkout -q --force --detach "$LLAMA_COMMIT" \
+        || die "拿不到 commit $LLAMA_COMMIT"
+    git -C "$SRC_DIR" reset -q --hard
+    rm -f "$SRC_DIR/.st-patched" "$SRC_DIR/.st-unpatched"
+}
+
+apply_patches() {
+    ensure_patches
+    [ -f "$SRC_DIR/.st-patched" ] && { say "patch 已套用"; return; }
+    local any=0
+    for p in "$PATCH_DIR"/*.patch; do
+        [ -e "$p" ] || continue
+        any=1
+        step "套用 $(basename "$p")"
+        # 已經套過就直接跳過（避免重跑時 patch 報錯）
+        if git -C "$SRC_DIR" apply --reverse --check "$p" >/dev/null 2>&1; then
+            say "  已套用，略過"
+            continue
+        fi
+        git -C "$SRC_DIR" apply --check "$p" 2>/dev/null \
+            || die "$(basename "$p") 套不上 llama.cpp $LLAMA_COMMIT（patch 過期了，要重新產生）"
+        git -C "$SRC_DIR" apply "$p" || die "$(basename "$p") 套用失敗"
+        say "  $(basename "$p")"
+    done
+    if [ "$any" = 1 ]; then touch "$SRC_DIR/.st-patched"; else touch "$SRC_DIR/.st-unpatched"; fi
+}
+
+# ============================================================= 4. 編譯
+build_if_needed() {
+    local server="$SRC_DIR/build/bin/llama-server"
+    if [ -x "$server" ] && { [ -f "$SRC_DIR/.st-patched" ] || [ -f "$SRC_DIR/.st-unpatched" ]; }; then
+        if [ -z "$(find "$SRC_DIR/src" -newer "$server" -name '*.cpp' -o -newer "$server" -name '*.h' 2>/dev/null | head -1)" ]; then
+            say "已經編譯好：$server"
+            return
+        fi
+    fi
+    step "編譯 llama-server（第一次會比較久）"
+    local jobs="${ST_BUILD_JOBS:-$(nproc 2>/dev/null || echo 2)}"
+    local gen=()
+    command -v ninja >/dev/null && gen=(-G Ninja)
+    mkdir -p "$LOG_DIR"
+    # 不開 -march=native：編譯機與執行機不一定同一台，native 會在別的機器 SIGILL。
+    cmake "${gen[@]}" -S "$SRC_DIR" -B "$SRC_DIR/build" \
+        -DCMAKE_BUILD_TYPE=Release -DGGML_NATIVE=OFF \
+        -DGGML_CURL=OFF -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_EXAMPLES=OFF \
+        -DLLAMA_BUILD_SERVER=ON -DLLAMA_BUILD_TOOLS=ON \
+        >"$LOG_DIR/cmake.log" 2>&1 || die "cmake 失敗，見 $LOG_DIR/cmake.log"
+    cmake --build "$SRC_DIR/build" --target llama-server -j "$jobs" \
+        >"$LOG_DIR/build.log" 2>&1 || die "編譯失敗，見 $LOG_DIR/build.log"
+    [ -x "$server" ] || die "編譯完但找不到 llama-server"
+    mkdir -p "$BIN_DIR"
+    cp -f "$server" "$BIN_DIR/llama-server"
+    say "編譯完成：$BIN_DIR/llama-server"
+}
+
+# ====================================================== 5. 下載模型權重
+ensure_model() {
+    mkdir -p "$MODEL_DIR"
+    local path="$MODEL_DIR/$MODEL_FILE"
+    if [ -f "$path" ]; then
+        local have
+        have=$(stat -c %s "$path")
+        if [ "$have" = "$MODEL_SIZE" ]; then
+            say "模型已經在硬碟上（$(( MODEL_SIZE / MIB )) MiB）"
+            printf '%s' "$path"; return
+        fi
+        say "模型檔大小不對（$have，應該是 $MODEL_SIZE），重新下載"
+        rm -f "$path"
+    fi
+    step "下載模型 $MODEL_FILE（$(( MODEL_SIZE / MIB )) MiB，可續傳）"
+    curl -fL -C - --retry 10 --retry-delay 5 --retry-all-errors \
+        -H "Authorization: Bearer ${HF_TOKEN:-}" \
+        -o "$path.part" \
+        "https://huggingface.co/$MODEL_REPO/resolve/main/$MODEL_FILE" \
+        || die "下載失敗，再跑一次會從斷點繼續"
+    local got
+    got=$(stat -c %s "$path.part")
+    [ "$got" = "$MODEL_SIZE" ] || die "大小不符：$got != $MODEL_SIZE（再跑一次續傳）"
+    mv -f "$path.part" "$path"
+    printf '%s' "$path"
+}
+
+# =========================================================== 6. 啟動參數
+build_args() {
+    SERVER_ARGS=(
+        --model "$MODEL_PATH"
+        --host 0.0.0.0
+        --port "$PORT"
+
+        -t "$THREADS"          # decode
+        -tb "$THREADS"         # prefill
+        -b "$UBATCH"           # n_ubatch
+
+        -c "$CTX"
+        --jinja                # 用 GGUF 內建的 chat template
+        --cache-reuse 256      # 多輪對話重用前綴
+
+        # 權重永遠走 file-backed mmap（不用 --mlock），所以 RAM 只放 arena。
+        # 分頁器的參數由 ST_* 環境變數帶進去（patch 0001）。
+    )
+}
+
+plan_report() {
+    cat >&2 <<EOF
+
+  設定摘要
+  ─────────────────────────────────────────────
+  模型        $MODEL_FILE（$(( MODEL_SIZE / MIB )) MiB，只在 SSD）
+  量化        Q4_K（不隨 RAM 改變）
+  架構        ${MODEL_ARCH}  ${N_LAYER} 層  hidden ${N_EMBD}  ${N_EXPERT} experts（用 ${N_EXPERT_USED}）
+  RAM 可用    ${USABLE_MB} MiB / cgroup 上限 ${CGROUP_MB} MiB
+  RAM 預算    ${RAM_BUDGET_MB} MiB（${BUDGET_SRC}）
+  非expert權重 ${NON_EXPERT_MIB} MiB（常駐 RAM）
+  KV 預留     ${KV_RESERVE_MB} MiB（ctx $CTX，f16；理論值 $KV_ESTIMATE_MB MiB，${KV_ELEMS} 元素/token）
+  compute     ${COMPUTE_RESERVE_MB} MiB
+  arena       ${ARENA_MB} MiB → 約 ${EXPERT_SLOTS} 個 expert 槽（expert 總量 ${EXPERT_TOTAL_MIB} MiB）
+  threads     $THREADS（配額 ${QUOTA:-無}／可用核心 $(affinity_cpus)）
+  ubatch      $UBATCH    ctx $CTX（模型上限 $MODEL_MAX_CTX）
+  GPU         ${GPU_MB} MiB
+  監聽        http://0.0.0.0:$PORT
+  環境        ST_RAM_BUDGET_MB=$RAM_BUDGET_MB ST_RESERVE_MB=$RESERVE_MB \
+              ST_KV_RESERVE_MB=$KV_RESERVE_MB ST_THREADS=$THREADS ST_UBATCH=$UBATCH \
+              ST_CTX=$CTX ST_IO_THREADS=$(( THREADS * 3 )) ST_PREFETCH=0
+  指令        ${SERVER_ARGS[*]}
+  ─────────────────────────────────────────────
+EOF
+}
+
+mkdir -p "$LOG_DIR"
+
+MODEL_PATH=""
+if [ "$PLAN_ONLY" = 0 ]; then
+    ensure_source
+    apply_patches
+    build_if_needed
+    MODEL_PATH=$(ensure_model)
+else
+    MODEL_PATH="$MODEL_DIR/$MODEL_FILE"
+    [ -x "$SRC_DIR/build/bin/llama-server" ] || say "尚未編譯（--plan 不會替你編）"
+fi
+
+[ -n "$MODEL_PATH" ] || MODEL_PATH="$MODEL_DIR/$MODEL_FILE"
+refresh_memory_budget
+build_args
+plan_report
+
+if [ "$PLAN_ONLY" = 1 ]; then
+    exit 0
+fi
+
+[ -f "$MODEL_PATH" ] || die "找不到模型 $MODEL_PATH"
+
+SERVER="$SRC_DIR/build/bin/llama-server"
+[ -x "$SERVER" ] || die "找不到 $SERVER"
+
+# 啟動前丟掉模型檔的 page cache，否則第一次量到的 SSD 讀取量會偏低。
+if [ -x "$SCRIPT_DIR/tools/drop_model_cache.py" ]; then
+    python3 "$SCRIPT_DIR/tools/drop_model_cache.py" "$MODEL_PATH" >/dev/null 2>&1 || true
+fi
+
+export ST_RAM_BUDGET_MB="$RAM_BUDGET_MB"
+export ST_RESERVE_MB="$RESERVE_MB"
+export ST_KV_RESERVE_MB="$KV_RESERVE_MB"
+export ST_THREADS="$THREADS"
+export ST_UBATCH="$UBATCH"
+export ST_CTX="$CTX"
+export ST_IO_THREADS=$(( THREADS * 3 ))
+[ "$ST_IO_THREADS" -gt 48 ] && ST_IO_THREADS=48
+
+step "啟動 llama-server（port $PORT）"
+if [ "$VERIFY" = 1 ]; then
+    "$SERVER" "${SERVER_ARGS[@]}" &
+    SRV_PID=$!
+    trap 'kill "$SRV_PID" 2>/dev/null' EXIT
+    for _ in $(seq 1 180); do
+        curl -fsS "http://127.0.0.1:$PORT/health" >/dev/null 2>&1 && break
+        sleep 2
+    done
+    python3 "$SCRIPT_DIR/tools/verify_run.py" --port "$PORT" \
+        --pid "$SRV_PID" --model "$MODEL_PATH" --out "$SCRIPT_DIR/validate/last-run.json" \
+        --ram-budget-mb "$RAM_BUDGET_MB" --arena-mb "$ARENA_MB" --expert-slots "$EXPERT_SLOTS"
+    exit $?
+fi
+
+exec "$SERVER" "${SERVER_ARGS[@]}"

sync.sh

@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+# ==============================================================================
+# sync.sh — 把工作成果推回 Hugging Face（agent 崩潰後的唯一保險）
+#
+#   ./sync.sh "訊息"     # 提交並推到 HelloSun/SmallThinker4b
+#   ./sync.sh --status   # 只看狀態
+#   ./sync.sh --pull     # 從遠端拉回最新（崩潰後續作第一步）
+#
+# 原則：**每改完程式並編譯過就上傳，不要累積。**
+# Token: 環境變數 HF_TOKEN，或檔案 .hf_token
+# ==============================================================================
+set -uo pipefail
+
+REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+STORAGE_REPO="${ST_STORAGE_REPO:-HelloSun/SmallThinker4b}"
+
+MODE=push
+MSG=""
+for a in "$@"; do
+    case "$a" in
+        --status) MODE=status ;;
+        --pull)   MODE=pull ;;
+        *)        MSG="$a" ;;
+    esac
+done
+[[ -z "$MSG" ]] && MSG="sync: $(date -Iseconds)"
+
+cd "$REPO_DIR" || { echo "[sync] 沒有 repo 目錄"; exit 1; }
+
+# ── token ──
+if [[ -z "${HF_TOKEN:-}" && -f "$REPO_DIR/.hf_token" ]]; then
+    HF_TOKEN="$(tr -d ' \n\r' < "$REPO_DIR/.hf_token")"
+fi
+if [[ -z "${HF_TOKEN:-}" ]]; then
+    echo "[sync] 找不到 HF_TOKEN"; exit 1
+fi
+export HF_TOKEN
+
+git config user.email "${HF_GIT_EMAIL:-agent@huggingface.co}"
+git config user.name "${HF_GIT_NAME:-Auto Upload Agent}"
+
+ensure_remote() {
+    local name="$1" repo="$2"
+    local url="https://oauth2:${HF_TOKEN}@huggingface.co/${repo}"
+    if git remote get-url "$name" >/dev/null 2>&1; then
+        git remote set-url "$name" "$url"
+    else
+        git remote add "$name" "$url"
+    fi
+}
+ensure_remote origin "$STORAGE_REPO"
+
+case "$MODE" in
+    status)
+        echo "── repo ──"; git remote -v
+        echo "── branch ──"; git status -sb | head -30
+        echo "── log ──"; git log --oneline -10
+        exit 0
+        ;;
+    pull)
+        git fetch origin
+        if git rev-parse HEAD >/dev/null 2>&1 && ! git diff --quiet 2>/dev/null; then
+            echo "[sync] 有未提交的改動，先 stash"; git stash push -u -m "sync-pull $(date -Iseconds)"
+        fi
+        git checkout -B main origin/main 2>/dev/null || git reset --hard origin/main
+        echo "[sync] 已拉到最新：$(git log --oneline -1)"
+        exit 0
+        ;;
+esac
+
+# ── push ──
+git add -A
+if git diff --cached --quiet; then
+    echo "[sync] 沒有改動，不用上傳"
+else
+    git commit -q -m "$MSG" || { echo "[sync] commit 失敗"; exit 1; }
+    # 第一次推送時遠端還沒有 main，要帶 -u 並允許無關歷史
+    git push -u origin main 2>&1 | tail -3 \
+        || git push -u --force origin main 2>&1 | tail -3
+fi
+
+echo "[sync] 完成：$(git log --oneline -1)"
+echo "[sync] https://huggingface.co/${STORAGE_REPO}"

tools/drop_model_cache.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+"""丟掉模型檔在系統 page cache 裡的頁（POSIX_FADV_DONTNEED）。
+
+為什麼需要：量 SSD 讀取量之前一定要做，否則「上一輪留在核心 page cache」
+會讓數字虛高（看起來像是 SSD 變快了，其實是記憶體）。
+
+用法：
+    python3 tools/drop_model_cache.py <model.gguf> [more files...]
+"""
+import os
+import sys
+
+POSIX_FADV_DONTNEED = 4
+SYNC = 2
+
+
+def drop(path: str) -> tuple[bool, str]:
+    try:
+        fd = os.open(path, os.O_RDONLY)
+    except OSError as e:
+        return False, str(e)
+    try:
+        size = os.fstat(fd).st_size
+        os.posix_fadvise(fd, 0, size, POSIX_FADV_DONTNEED)
+        os.fsync(fd)
+        return True, f"{size} bytes"
+    except (AttributeError, OSError) as e:
+        return False, str(e)
+    finally:
+        os.close(fd)
+
+
+def main() -> int:
+    if len(sys.argv) < 2:
+        print(__doc__)
+        return 2
+    rc = 0
+    for p in sys.argv[1:]:
+        if not os.path.exists(p):
+            print(f"[drop] 找不到 {p}")
+            rc = 1
+            continue
+        ok, info = drop(p)
+        print(f"[drop] {'ok  ' if ok else 'fail'} {p} {info}")
+        if not ok:
+            rc = 1
+    return rc
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tools/io_probe.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+"""實測這台機器的 SSD 讀取能力。
+
+**換機器一定要重測這個數字**，tok/s 幾乎完全由它決定（dense/每 token 重讀整個權重）。
+輸出 JSON 給 validate/io-probe.json。
+
+    python3 tools/io_probe.py --out validate/io-probe.json --size-mib 1024
+"""
+import argparse
+import json
+import os
+import sys
+import time
+
+MIB = 1024 * 1024
+
+
+def drop_cache(fd: int, size: int) -> None:
+    os.posix_fadvise(fd, 0, size, 4)  # POSIX_FADV_DONTNEED
+    os.fsync(fd)
+
+
+def read_seq(path: str, size: int, threads: int) -> dict:
+    """threads 條執行緒各自讀不同區段，量「有效頻寬」。"""
+    import threading
+    per = size // threads
+    results = [0] * threads
+    fds = [os.open(path, os.O_RDONLY) for _ in range(threads)]
+    for fd in fds:
+        drop_cache(fd, size)
+
+    def work(i: int):
+        fd = fds[i]
+        off = i * per
+        left = per
+        chunk = 4 * MIB
+        got = 0
+        while left > 0:
+            n = os.pread(fd, min(chunk, left), off + got)
+            if not n:
+                break
+            got += n
+            left -= n
+        results[i] = got
+
+    ts = [threading.Thread(target=work, args=(i,)) for i in range(threads)]
+    t0 = time.time()
+    for t in ts:
+        t.start()
+    for t in ts:
+        t.join()
+    dt = time.time() - t0
+    for fd in fds:
+        os.close(fd)
+    total = sum(results)
+    return {"threads": threads, "bytes": total, "seconds": round(dt, 3),
+            "mb_per_s": round(total / MIB / dt, 1) if dt > 0 else None}
+
+
+def read_rand4k(path: str, count: int) -> dict:
+    fd = os.open(path, os.O_RDONLY)
+    size = os.fstat(fd).st_size
+    drop_cache(fd, size)
+    step = max(size // (count + 1), 4096)
+    buf = 4096
+    t0 = time.time()
+    for i in range(1, count + 1):
+        os.pread(fd, buf, i * step)
+    dt = time.time() - t0
+    os.close(fd)
+    return {"count": count, "seconds": round(dt, 3), "iops": round(count / dt, 1) if dt else None}
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--path", default=None, help="測試檔；沒給就用模型檔")
+    ap.add_argument("--model", default=None)
+    ap.add_argument("--out", required=True)
+    ap.add_argument("--size-mib", type=int, default=1024)
+    ap.add_argument("--ramp", type=int, default=512, help="隨機讀的次數")
+    a = ap.parse_args()
+
+    path = a.path
+    tmp = None
+    if not path:
+        m = a.model or os.environ.get("ST_MODEL_PATH")
+        if not m:
+            print("需要 --path 或 --model（或 ST_MODEL_PATH）", file=sys.stderr)
+            return 2
+        path = m
+    if not os.path.exists(path):
+        tmp = f"/tmp/io-probe-{os.getpid()}.bin"
+        size = a.size_mib * MIB
+        print(f"[io_probe] 造 {a.size_mib} MiB 測試檔 {tmp}", file=sys.stderr)
+        with open(tmp, "wb") as f:
+            f.write(os.urandom(size))
+        path = tmp
+
+    size = min(a.size_mib * MIB, os.path.getsize(path))
+    seq = [read_seq(path, size, t) for t in (1, 2, 4, 8) if size // t >= 8 * MIB]
+    res = {
+        "when": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
+        "file": path,
+        "size_mib": size // MIB,
+        "fs": os.statvfs(path).f_bsize,
+        "sequential": seq,
+        "random_4k": read_rand4k(path, a.ramp),
+        "cpu_count": os.cpu_count(),
+    }
+    if tmp:
+        res["tmp_removed"] = tmp
+        os.unlink(tmp)
+    os.makedirs(os.path.dirname(os.path.abspath(a.out)) or ".", exist_ok=True)
+    with open(a.out, "w") as f:
+        json.dump(res, f, indent=2)
+        f.write("\n")
+    print(json.dumps(res, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tools/verify_run.py

@@ -0,0 +1,200 @@
+#!/usr/bin/env python3
+"""對已經在跑的 llama-server 打一次 chat request，同時量記憶體。
+
+量到的东西（每 0.2s 取樣一次，取峰值）：
+  - total RSS      /proc/<pid>/status 的 VmRSS（**包含 file-backed mmap 的權重頁**）
+  - anon RSS       smaps_rollup 的 Anonymous，扣掉 page cache 後的「真的在用 RAM」
+  - file RSS       smaps_rollup 的 Private_Dirty + Private_Clean（權重對映）
+  - swap           /proc/<pid>/status 的 VmSwap（必須是 0）
+  - io             /proc/<pid>/io 的 read_bytes（真正落到 block device 的量）
+
+為什麼只看 total RSS：GGUF 是 mmap(MAP_SHARED) 對映的，權重頁會算進 RSS
+而且真的佔用系統 RAM。只看匿名記憶體會嚴重低估。這個坑本專案第一版就踩過。
+
+用法：
+    python3 tools/verify_run.py --port 8080 --pid 12345 --model path.gguf --out out.json
+"""
+import argparse
+import json
+import os
+import subprocess
+import sys
+import threading
+import time
+import urllib.error
+import urllib.request
+
+MIB = 1024 * 1024
+
+
+def read_status(pid: int) -> dict:
+    out = {}
+    try:
+        with open(f"/proc/{pid}/status") as f:
+            for line in f:
+                k, _, v = line.partition(":")
+                if k in ("VmRSS", "VmSwap", "VmHWM"):
+                    out[k] = int(v.split()[0]) * 1024
+    except (OSError, ValueError, IndexError):
+        pass
+    return out
+
+
+def read_smaps(pid: int) -> dict:
+    out = {"Anonymous": 0, "Private_Dirty": 0, "Private_Clean": 0, "Shared_Clean": 0}
+    try:
+        with open(f"/proc/{pid}/smaps_rollup") as f:
+            for line in f:
+                k, _, v = line.partition(":")
+                if k in out:
+                    out[k] = int(v.split()[0]) * 1024
+    except (OSError, ValueError, IndexError):
+        pass
+    return out
+
+
+def read_io(pid: int) -> dict:
+    out = {}
+    try:
+        with open(f"/proc/{pid}/io") as f:
+            for line in f:
+                k, _, v = line.partition(":")
+                out[k] = int(v.strip())
+    except (OSError, ValueError):
+        pass
+    return out
+
+
+class Sampler(threading.Thread):
+    def __init__(self, pid: int, interval: float = 0.2):
+        super().__init__(daemon=True)
+        self.pid = pid
+        self.interval = interval
+        self.stop_flag = threading.Event()
+        self.peak = {"total_rss": 0, "anon_rss": 0, "file_rss": 0, "swap": 0, "hwm_rss": 0}
+        self.io0 = None
+        self.io1 = None
+
+    def run(self):
+        self.io0 = read_io(self.pid)
+        while not self.stop_flag.is_set():
+            st = read_status(self.pid)
+            sm = read_smaps(self.pid)
+            anon = sm["Anonymous"]
+            filed = sm["Private_Dirty"] + sm["Private_Clean"]
+            self.peak["total_rss"] = max(self.peak["total_rss"], st.get("VmRSS", 0))
+            self.peak["anon_rss"] = max(self.peak["anon_rss"], anon)
+            self.peak["file_rss"] = max(self.peak["file_rss"], filed)
+            self.peak["swap"] = max(self.peak["swap"], st.get("VmSwap", 0))
+            self.peak["hwm_rss"] = max(self.peak["hwm_rss"], st.get("VmHWM", 0))
+            self.stop_flag.wait(self.interval)
+        self.io1 = read_io(self.pid)
+
+
+def wait_health(port: int, timeout: float) -> bool:
+    end = time.time() + timeout
+    while time.time() < end:
+        try:
+            with urllib.request.urlopen(f"http://127.0.0.1:{port}/health", timeout=5) as r:
+                if r.status == 200:
+                    return True
+        except (urllib.error.URLError, OSError):
+            time.sleep(2)
+    return False
+
+
+def chat(port: int, prompt: str, max_tokens: int, timeout: float) -> dict:
+    body = json.dumps({
+        "messages": [{"role": "user", "content": prompt}],
+        "max_tokens": max_tokens,
+        "temperature": 0.0,
+    }).encode()
+    req = urllib.request.Request(
+        f"http://127.0.0.1:{port}/v1/chat/completions",
+        data=body, headers={"Content-Type": "application/json"}, method="POST")
+    t0 = time.time()
+    with urllib.request.urlopen(req, timeout=timeout) as r:
+        data = json.loads(r.read())
+    dt = time.time() - t0
+    usage = data.get("usage", {})
+    n = usage.get("completion_tokens", 0)
+    return {
+        "prompt_tokens": usage.get("prompt_tokens"),
+        "completion_tokens": n,
+        "seconds": round(dt, 2),
+        "tok_per_s": round(n / dt, 4) if dt > 0 and n else None,
+        "content": data["choices"][0]["message"]["content"],
+        "timings": data.get("timings"),
+    }
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--port", type=int, required=True)
+    ap.add_argument("--pid", type=int, required=True)
+    ap.add_argument("--model", required=True)
+    ap.add_argument("--out", required=True)
+    ap.add_argument("--prompt", default="Explain in one sentence what a MoE layer does.")
+    ap.add_argument("--max-tokens", type=int, default=16)
+    ap.add_argument("--timeout", type=float, default=1800)
+    ap.add_argument("--ram-budget-mb", type=int, default=0)
+    ap.add_argument("--arena-mb", type=int, default=0)
+    ap.add_argument("--expert-slots", type=int, default=0)
+    a = ap.parse_args()
+
+    result = {
+        "when": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
+        "model": a.model,
+        "port": a.port,
+        "pid": a.pid,
+        "ram_budget_mb": a.ram_budget_mb,
+        "arena_mb": a.arena_mb,
+        "expert_slots": a.expert_slots,
+    }
+
+    if not wait_health(a.port, 300):
+        result["error"] = "server 沒有在 300 秒內 ready"
+        _write(a.out, result)
+        return 1
+
+    s = Sampler(a.pid)
+    s.start()
+    try:
+        result["chat"] = chat(a.port, a.prompt, a.max_tokens, a.timeout)
+    except (urllib.error.URLError, OSError, ValueError, KeyError) as e:
+        result["error"] = f"chat failed: {e}"
+    time.sleep(2)  # 讓峰值取樣涵蓋到收尾
+    s.stop_flag.set()
+    s.join(timeout=10)
+
+    io_delta = {}
+    if s.io0 and s.io1:
+        for k in ("read_bytes", "rchar", "write_bytes"):
+            if k in s.io0 and k in s.io1:
+                io_delta[k] = s.io1[k] - s.io0[k]
+
+    result["peak"] = {
+        "total_rss_gb": round(s.peak["total_rss"] / 1024 ** 3, 4),
+        "anon_rss_gb": round(s.peak["anon_rss"] / 1024 ** 3, 4),
+        "file_rss_gb": round(s.peak["file_rss"] / 1024 ** 3, 4),
+        "peak_swap_gb": round(s.peak["swap"] / 1024 ** 3, 4),
+        "hwm_rss_gb": round(s.peak["hwm_rss"] / 1024 ** 3, 4),
+    }
+    result["io_delta"] = io_delta
+    result["model_size_mib"] = os.path.getsize(a.model) // MIB if os.path.exists(a.model) else None
+    ok = "error" not in result and result["peak"]["peak_swap_gb"] == 0
+    result["ok"] = ok
+    _write(a.out, result)
+    print(json.dumps(result, indent=2, ensure_ascii=False))
+    return 0 if ok else 1
+
+
+def _write(path: str, obj: dict) -> None:
+    os.makedirs(os.path.dirname(os.path.abspath(path)) or ".", exist_ok=True)
+    with open(path, "w") as f:
+        json.dump(obj, f, indent=2, ensure_ascii=False)
+        f.write("\n")
+
+
+if __name__ == "__main__":
+    sys.exit(main())