Pipeline stage P2: explain and fix refiner Q8_0 slowdown, V-pack hoist + 128x128 flash-attn tiles

111541a 1 day ago

11.2 kB

#!/usr/bin/env bash
# SmartTest.sh - generate the same cat+snake image with every built variant and write a speed report.
#
#   ./SmartTest.sh                    # all rows of the matrix, 1 repeat
#   REPEAT=3 ./SmartTest.sh           # median of 3 runs per row
#   ./SmartTest.sh --only Q8          # only rows whose name contains "Q8"
#
# Output (in results/):
#   results.csv      one line per run
#   results.md       comparison table (medians), ready for the model card
#   hardware.txt     machine configuration used for the run
#   images/*.png     generated 512x512 images
#   logs/*.log       raw sd-cli logs
set -euo pipefail

ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
WORK="${WORK:-$PWD/work}"
BIN="$WORK/bin"
MODELS="$WORK/models"
OUT="${OUT:-$ROOT/results}"
PY="$WORK/venv/bin/python"; [[ -x "$PY" ]] || PY="$(command -v python3)"

THREADS="${THREADS:-$(nproc)}"
CPUSET="${CPUSET:-0-$((THREADS - 1))}"
REPEAT="${REPEAT:-1}"
ONLY="${ONLY:-}"   # env ONLY=<substring>, or argument: --only <substring>
[[ "${1:-}" == "--only" ]] && ONLY="${2:?--only needs a substring}"

SEED=42
STEPS=8
WIDTH=512
HEIGHT=512
PROMPT="cute cartoon cat and cute cartoon snake, best friends, hugging together, kawaii, simple clean background, children's book illustration, soft colors, adorable"
NEGATIVE="blurry, low quality, realistic, photo"

# name | variant | model key | extra env (- = none)
MATRIX=(
    "stock-Q4_K|stock|Q4_K|-"
    "avx2-Q4_K|avx2|Q4_K|-"
    "avx512-Q4_K|avx512|Q4_K|-"
    "amx_lto-Q4_K|amx_lto|Q4_K|-"
    "amx_lto-Q4_K+AMXw|amx_lto|Q4_K|SD_CPU_EXTRA_BUFT=1"
    "amx_lto-Q4_K-ffn13|amx_lto|Q4_K-ffn13|-"
    "amx_lto-Q4_K-ffn13+AMXw|amx_lto|Q4_K-ffn13|SD_CPU_EXTRA_BUFT=1"
    "amx_lto-Q4_0|amx_lto|Q4_0|-"
    "amx_lto-Q8_0|amx_lto|Q8_0|-"
    "amx_lto-Q8_0+AMXw|amx_lto|Q8_0|SD_CPU_EXTRA_BUFT=1"
    # stage 1 (VAE decode): llamafile sgemm build, with and without conv-direct for the VAE
    "llamafile-Q4_K-ffn13+AMXw|llamafile|Q4_K-ffn13|SD_CPU_EXTRA_BUFT=1|"
    "llamafile-Q4_K-ffn13+AMXw+convdirect|llamafile|Q4_K-ffn13|SD_CPU_EXTRA_BUFT=1|--conv-direct auto,vae=on"
    # stage P2: refiner weights requantized Q8_0 -> Q4_K (speed-first, opt-in)
    "llamafile-Q4_K-ffn13-rq+AMXw+convdirect|llamafile|Q4_K-ffn13-rq|SD_CPU_EXTRA_BUFT=1|--conv-direct auto,vae=on"
)

model_file() {
    case "$1" in
        Q4_K)       echo "$MODELS/z_image_turbo-Q4_K.gguf" ;;
        Q4_K-ffn13) echo "$MODELS/z_image_turbo-Q4_K-ffn13.gguf" ;;
        Q4_K-ffn13-rq) echo "$MODELS/z_image_turbo-Q4_K-ffn13-rq.gguf" ;;
        Q4_0)       echo "$MODELS/z_image_turbo-Q4_0.gguf" ;;
        Q8_0)       echo "$MODELS/z_image_turbo-Q8_0.gguf" ;;
        *) return 1 ;;
    esac
}

log() { printf '\033[1;32m[smarttest]\033[0m %s\n' "$*"; }

mkdir -p "$OUT/images" "$OUT/logs"
CSV="$OUT/results.csv"
# APPEND=1 keeps existing rows in results.csv (additive testing: only new rows are run).
if [[ "${APPEND:-0}" != "1" || ! -s "$CSV" ]]; then
    echo "name,variant,model,env,rep,status,sampling_s,generate_s,wall_s,peak_rss_gib,diff_mean_vs_ref,diff_max_vs_ref,image" > "$CSV"
fi

# ------------------------------------------------------------ hardware report
{
    echo "date:        $(date -u '+%Y-%m-%d %H:%M:%S UTC')"
    echo "os:          $(. /etc/os-release && echo "$PRETTY_NAME")"
    echo "kernel:      $(uname -r)"
    echo "cpu:         $(lscpu | awk -F: '/^Model name/ {sub(/^ +/,"",$2); print $2}')"
    echo "sockets:     $(lscpu | awk -F: '/^Socket/ {sub(/^ +/,"",$2); print $2}')"
    echo "cores/threads per socket: $(lscpu | awk -F: '/^Core\(s\) per socket/ {sub(/^ +/,"",$2); print $2}') / $(lscpu | awk -F: '/^Thread\(s\) per core/ {sub(/^ +/,"",$2); print $2}') SMT"
    echo "numa nodes:  $(lscpu | awk -F: '/^NUMA node\(s\)/ {sub(/^ +/,"",$2); print $2}')"
    echo "cpu flags:   $(grep -o -E 'avx512f|avx512_vnni|avx512_bf16|amx_int8|amx_bf16|avx2' /proc/cpuinfo | sort -u | tr '\n' ' ')"
    echo "cgroup cpu quota: $(cat /sys/fs/cgroup/cpu.max 2>/dev/null || echo n/a)"
    echo "threads used: $THREADS   cpuset: $CPUSET"
    echo "memory:      $(free -g | awk '/^Mem:/ {print $2" GiB total"}')"
    echo "gpu:         $(nvidia-smi -L 2>&1 | head -1)  (no GPU backend used; CPU-only inference)"
    echo "gcc:         $(gcc -dumpfullversion 2>/dev/null || echo n/a)"
    echo "cmake:       $(cmake --version 2>/dev/null | head -1 || echo n/a)"
    echo "sd.cpp:      $(git -C "$WORK/stable-diffusion.cpp" rev-parse HEAD 2>/dev/null || echo n/a)"
    echo "openvino env: $("$WORK/venv-ov/bin/python" -c 'import openvino, optimum.intel, diffusers; print(f"openvino {openvino.__version__}, optimum-intel {optimum.intel.__version__}, diffusers {diffusers.__version__}")' 2>/dev/null || echo n/a)"
    echo "openvino model: HelloSun/Z-Image-Turbo-OpenVINO-INT4 (transformer/text_encoder INT4, VAE INT8)"
    echo "prompt:      $PROMPT"
    echo "negative:    $NEGATIVE"
    echo "size/steps/seed/cfg: ${WIDTH}x${HEIGHT} / $STEPS / $SEED / 1.0 (euler, simple scheduler)"
} > "$OUT/hardware.txt"

# ------------------------------------------------------------ run cases
REF_IMAGE=""
[[ -f "$OUT/images/stock-Q4_K-r1.png" ]] && REF_IMAGE="$OUT/images/stock-Q4_K-r1.png"
OV_DIR="$MODELS/Z-Image-Turbo-OpenVINO-INT4"
OV_PY="$WORK/venv-ov/bin/python"

# run_case <name> <variant> <model-label> <env-spec> <rep-command...>
# The rep-command is executed under taskset; "__IMG__" is replaced by the output image path.
# It must print sd-cli style timing lines (sampling / generate_image) on stderr.
run_case() {
    local name="$1" variant="$2" mlabel="$3" envspec="$4"; shift 4
    local extra=()
    if [[ -n "${RUN_EXTRA:-}" ]]; then read -ra extra <<< "$RUN_EXTRA"; fi
    local envprefix=()
    [[ "$envspec" != "-" ]] && envprefix=(env "$envspec")
    for rep in $(seq 1 "$REPEAT"); do
        local tag="${name}-r${rep}"
        local log_file="$OUT/logs/$tag.log"
        local img="$OUT/images/$tag.png"
        log "run $tag"
        local cmd=() a
        for a in "$@" "${extra[@]}"; do
            if [[ "$a" == "__IMG__" ]]; then cmd+=("$img"); else cmd+=("$a"); fi
        done
        # stdout and stderr of the generator both go to the log; measure_rss appends its summary line too.
        set +e
        "${envprefix[@]}" "$PY" "$ROOT/tools/measure_rss.py" taskset -c "$CPUSET" "${cmd[@]}" \
            > "$log_file" 2>&1
        local rc=$?
        set -e
        local rss_line sampling generate wall
        rss_line="$(grep -oE 'PEAK_RSS=[0-9.]+' "$log_file" | cut -d= -f2 | tail -1 || true)"
        sampling="$(grep -oE 'sampling completed, taking [0-9.]+' "$log_file" | grep -oE '[0-9.]+' | tail -1 || true)"
        generate="$(grep -oE 'generate_image completed in [0-9.]+' "$log_file" | grep -oE '[0-9.]+' | tail -1 || true)"
        wall="$(grep -oE 'WALL=[0-9.]+' "$log_file" | cut -d= -f2 | tail -1 || true)"

        local status="ok" diff_mean="" diff_max=""
        if [[ $rc -ne 0 || ! -s "$img" ]]; then
            status="failed(rc=$rc)"
        else
            [[ -z "$REF_IMAGE" ]] && REF_IMAGE="$img"
            read -r diff_mean diff_max < <("$PY" - "$REF_IMAGE" "$img" <<'PY'
import sys
import numpy as np
from PIL import Image
a = np.asarray(Image.open(sys.argv[1]).convert("RGB")).astype(int)
b = np.asarray(Image.open(sys.argv[2]).convert("RGB")).astype(int)
d = np.abs(a - b)
print(f"{d.mean():.4f} {d.max()}")
PY
)
        fi
        echo "$name,$variant,$mlabel,$envspec,$rep,$status,$sampling,$generate,$wall,$rss_line,$diff_mean,$diff_max,images/$tag.png" >> "$CSV"
        log "  $status sampling=${sampling:-?}s generate=${generate:-?}s rss=${rss_line:-?}GiB diff_mean=${diff_mean:-n/a}"
    done
}

# ---- sd.cpp matrix
for row in "${MATRIX[@]}"; do
    IFS='|' read -r name variant mkey envspec extra_args <<< "$row"
    if [[ -n "$ONLY" && "$name" != *"$ONLY"* ]]; then continue; fi

    sdbin="$BIN/$variant/sd-cli"
    model="$(model_file "$mkey")"
    if [[ ! -x "$sdbin" || ! -f "$model" ]]; then
        log "skip $name (missing binary or model)"
        echo "$name,$variant,$mkey,$envspec,0,missing,,,,,,,," >> "$CSV"
        continue
    fi
    RUN_EXTRA="${extra_args:-}"
    run_case "$name" "$variant" "$mkey" "$envspec" \
        "$sdbin" \
        --diffusion-model "$model" \
        --vae "$MODELS/ae.safetensors" \
        --llm "$MODELS/Qwen3-4B-Instruct-2507-Q4_K_M.gguf" \
        -p "$PROMPT" -n "$NEGATIVE" \
        -W "$WIDTH" -H "$HEIGHT" --steps "$STEPS" --cfg-scale 1.0 \
        --seed "$SEED" --sampling-method euler --scheduler simple \
        -t "$THREADS" -o "__IMG__"
done

# ---- OpenVINO INT4 (HelloSun/Z-Image-Turbo-OpenVINO-INT4), 8 steps to match sd.cpp; same prompt/seed/size
for ov_row in "ov-int4-8step|8"; do
    IFS='|' read -r name steps <<< "$ov_row"
    if [[ -n "$ONLY" && "$name" != *"$ONLY"* ]]; then continue; fi
    if [[ ! -x "$OV_PY" || ! -d "$OV_DIR" ]]; then
        log "skip $name (missing OpenVINO venv or model; see build.sh)"
        echo "$name,openvino,OV-INT4,-,0,missing,,,,,,,," >> "$CSV"
        continue
    fi
    run_case "$name" "openvino" "OV-INT4" "-" \
        "$OV_PY" "$ROOT/tools/ov_bench.py" \
        --model-dir "$OV_DIR" -p "$PROMPT" \
        -W "$WIDTH" -H "$HEIGHT" --steps "$steps" --seed "$SEED" -o "__IMG__"
done

# ------------------------------------------------------------ report (medians)
"$PY" - "$CSV" "$OUT/results.md" <<'PY'
import csv, statistics, sys
from collections import OrderedDict

rows = list(csv.DictReader(open(sys.argv[1])))
groups = OrderedDict()
for r in rows:
    if r["status"] != "ok":
        continue
    groups.setdefault(r["name"], []).append(r)

def med(values):
    vals = [float(v) for v in values if v not in ("", None)]
    return statistics.median(vals) if vals else None

ref = None
if "stock-Q4_K" in groups:
    ref = med([r["generate_s"] for r in groups["stock-Q4_K"]])

lines = [
    "| # | 版本 (variant) | 模型 | 額外環境變數 | 採樣 (s) | 總生成 (s) | 相對 stock | 峰值 RSS (GiB) | 與 stock 像素差 (mean/max) |",
    "|---|---|---|---|---|---|---|---|---|",
]
for i, (name, rs) in enumerate(groups.items(), 1):
    samp = med([r["sampling_s"] for r in rs])
    gen = med([r["generate_s"] for r in rs])
    rss = med([r["peak_rss_gib"] for r in rs])
    dm = med([r["diff_mean_vs_ref"] for r in rs])
    dmx = med([r["diff_max_vs_ref"] for r in rs])
    f1 = lambda v: "n/a" if v is None else f"{v:.1f}"
    f2 = lambda v: "n/a" if v is None else f"{v:.2f}"
    speed = f"{ref / gen:.2f}x" if ref and gen else "n/a"
    diff = "ref" if name == "stock-Q4_K" else (f"{dm:.2f} / {dmx:.0f}" if dm is not None else "n/a")
    env = rs[0]["env"] if rs[0]["env"] != "-" else "-"
    if "convdirect" in name:
        env = (env + " + conv-direct") if env != "-" else "conv-direct"
    lines.append(f"| {i} | {rs[0]['variant']} | {rs[0]['model']} | {env} | {f1(samp)} | {f1(gen)} | {speed} | {f2(rss)} | {diff} |")
lines.append("")
lines.append(f"n = {max((len(v) for v in groups.values()), default=0)} run(s) per row (median). Speedup = stock-Q4_K generate time / row generate time.")
open(sys.argv[2], "w").write("\n".join(lines) + "\n")
print("\n".join(lines))
PY

log "results: $CSV"
log "table:   $OUT/results.md"

	#!/usr/bin/env bash
	# SmartTest.sh - generate the same cat+snake image with every built variant and write a speed report.
	#
	# ./SmartTest.sh # all rows of the matrix, 1 repeat
	# REPEAT=3 ./SmartTest.sh # median of 3 runs per row
	# ./SmartTest.sh --only Q8 # only rows whose name contains "Q8"
	#
	# Output (in results/):
	# results.csv one line per run
	# results.md comparison table (medians), ready for the model card
	# hardware.txt machine configuration used for the run
	# images/*.png generated 512x512 images
	# logs/*.log raw sd-cli logs
	set -euo pipefail

	ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
	WORK="${WORK:-$PWD/work}"
	BIN="$WORK/bin"
	MODELS="$WORK/models"
	OUT="${OUT:-$ROOT/results}"
	PY="$WORK/venv/bin/python"; [[ -x "$PY" ]] \|\| PY="$(command -v python3)"

	THREADS="${THREADS:-$(nproc)}"
	CPUSET="${CPUSET:-0-$((THREADS - 1))}"
	REPEAT="${REPEAT:-1}"
	ONLY="${ONLY:-}" # env ONLY=<substring>, or argument: --only <substring>
	[[ "${1:-}" == "--only" ]] && ONLY="${2:?--only needs a substring}"

	SEED=42
	STEPS=8
	WIDTH=512
	HEIGHT=512
	PROMPT="cute cartoon cat and cute cartoon snake, best friends, hugging together, kawaii, simple clean background, children's book illustration, soft colors, adorable"
	NEGATIVE="blurry, low quality, realistic, photo"

	# name \| variant \| model key \| extra env (- = none)
	MATRIX=(
	"stock-Q4_K\|stock\|Q4_K\|-"
	"avx2-Q4_K\|avx2\|Q4_K\|-"
	"avx512-Q4_K\|avx512\|Q4_K\|-"
	"amx_lto-Q4_K\|amx_lto\|Q4_K\|-"
	"amx_lto-Q4_K+AMXw\|amx_lto\|Q4_K\|SD_CPU_EXTRA_BUFT=1"
	"amx_lto-Q4_K-ffn13\|amx_lto\|Q4_K-ffn13\|-"
	"amx_lto-Q4_K-ffn13+AMXw\|amx_lto\|Q4_K-ffn13\|SD_CPU_EXTRA_BUFT=1"
	"amx_lto-Q4_0\|amx_lto\|Q4_0\|-"
	"amx_lto-Q8_0\|amx_lto\|Q8_0\|-"
	"amx_lto-Q8_0+AMXw\|amx_lto\|Q8_0\|SD_CPU_EXTRA_BUFT=1"
	# stage 1 (VAE decode): llamafile sgemm build, with and without conv-direct for the VAE
	"llamafile-Q4_K-ffn13+AMXw\|llamafile\|Q4_K-ffn13\|SD_CPU_EXTRA_BUFT=1\|"
	"llamafile-Q4_K-ffn13+AMXw+convdirect\|llamafile\|Q4_K-ffn13\|SD_CPU_EXTRA_BUFT=1\|--conv-direct auto,vae=on"
	# stage P2: refiner weights requantized Q8_0 -> Q4_K (speed-first, opt-in)
	"llamafile-Q4_K-ffn13-rq+AMXw+convdirect\|llamafile\|Q4_K-ffn13-rq\|SD_CPU_EXTRA_BUFT=1\|--conv-direct auto,vae=on"
	)

	model_file() {
	case "$1" in
	Q4_K) echo "$MODELS/z_image_turbo-Q4_K.gguf" ;;
	Q4_K-ffn13) echo "$MODELS/z_image_turbo-Q4_K-ffn13.gguf" ;;
	Q4_K-ffn13-rq) echo "$MODELS/z_image_turbo-Q4_K-ffn13-rq.gguf" ;;
	Q4_0) echo "$MODELS/z_image_turbo-Q4_0.gguf" ;;
	Q8_0) echo "$MODELS/z_image_turbo-Q8_0.gguf" ;;
	*) return 1 ;;
	esac
	}

	log() { printf '\033[1;32m[smarttest]\033[0m %s\n' "$*"; }

	mkdir -p "$OUT/images" "$OUT/logs"
	CSV="$OUT/results.csv"
	# APPEND=1 keeps existing rows in results.csv (additive testing: only new rows are run).
	if [[ "${APPEND:-0}" != "1" \|\| ! -s "$CSV" ]]; then
	echo "name,variant,model,env,rep,status,sampling_s,generate_s,wall_s,peak_rss_gib,diff_mean_vs_ref,diff_max_vs_ref,image" > "$CSV"
	fi

	# ------------------------------------------------------------ hardware report
	{
	echo "date: $(date -u '+%Y-%m-%d %H:%M:%S UTC')"
	echo "os: $(. /etc/os-release && echo "$PRETTY_NAME")"
	echo "kernel: $(uname -r)"
	echo "cpu: $(lscpu \| awk -F: '/^Model name/ {sub(/^ +/,"",$2); print $2}')"
	echo "sockets: $(lscpu \| awk -F: '/^Socket/ {sub(/^ +/,"",$2); print $2}')"
	echo "cores/threads per socket: $(lscpu \| awk -F: '/^Core\(s\) per socket/ {sub(/^ +/,"",$2); print $2}') / $(lscpu \| awk -F: '/^Thread\(s\) per core/ {sub(/^ +/,"",$2); print $2}') SMT"
	echo "numa nodes: $(lscpu \| awk -F: '/^NUMA node\(s\)/ {sub(/^ +/,"",$2); print $2}')"
	echo "cpu flags: $(grep -o -E 'avx512f\|avx512_vnni\|avx512_bf16\|amx_int8\|amx_bf16\|avx2' /proc/cpuinfo \| sort -u \| tr '\n' ' ')"
	echo "cgroup cpu quota: $(cat /sys/fs/cgroup/cpu.max 2>/dev/null \|\| echo n/a)"
	echo "threads used: $THREADS cpuset: $CPUSET"
	echo "memory: $(free -g \| awk '/^Mem:/ {print $2" GiB total"}')"
	echo "gpu: $(nvidia-smi -L 2>&1 \| head -1) (no GPU backend used; CPU-only inference)"
	echo "gcc: $(gcc -dumpfullversion 2>/dev/null \|\| echo n/a)"
	echo "cmake: $(cmake --version 2>/dev/null \| head -1 \|\| echo n/a)"
	echo "sd.cpp: $(git -C "$WORK/stable-diffusion.cpp" rev-parse HEAD 2>/dev/null \|\| echo n/a)"
	echo "openvino env: $("$WORK/venv-ov/bin/python" -c 'import openvino, optimum.intel, diffusers; print(f"openvino {openvino.__version__}, optimum-intel {optimum.intel.__version__}, diffusers {diffusers.__version__}")' 2>/dev/null \|\| echo n/a)"
	echo "openvino model: HelloSun/Z-Image-Turbo-OpenVINO-INT4 (transformer/text_encoder INT4, VAE INT8)"
	echo "prompt: $PROMPT"
	echo "negative: $NEGATIVE"
	echo "size/steps/seed/cfg: ${WIDTH}x${HEIGHT} / $STEPS / $SEED / 1.0 (euler, simple scheduler)"
	} > "$OUT/hardware.txt"

	# ------------------------------------------------------------ run cases
	REF_IMAGE=""
	[[ -f "$OUT/images/stock-Q4_K-r1.png" ]] && REF_IMAGE="$OUT/images/stock-Q4_K-r1.png"
	OV_DIR="$MODELS/Z-Image-Turbo-OpenVINO-INT4"
	OV_PY="$WORK/venv-ov/bin/python"

	# run_case <name> <variant> <model-label> <env-spec> <rep-command...>
	# The rep-command is executed under taskset; "__IMG__" is replaced by the output image path.
	# It must print sd-cli style timing lines (sampling / generate_image) on stderr.
	run_case() {
	local name="$1" variant="$2" mlabel="$3" envspec="$4"; shift 4
	local extra=()
	if [[ -n "${RUN_EXTRA:-}" ]]; then read -ra extra <<< "$RUN_EXTRA"; fi
	local envprefix=()
	[[ "$envspec" != "-" ]] && envprefix=(env "$envspec")
	for rep in $(seq 1 "$REPEAT"); do
	local tag="${name}-r${rep}"
	local log_file="$OUT/logs/$tag.log"
	local img="$OUT/images/$tag.png"
	log "run $tag"
	local cmd=() a
	for a in "$@" "${extra[@]}"; do
	if [[ "$a" == "__IMG__" ]]; then cmd+=("$img"); else cmd+=("$a"); fi
	done
	# stdout and stderr of the generator both go to the log; measure_rss appends its summary line too.
	set +e
	"${envprefix[@]}" "$PY" "$ROOT/tools/measure_rss.py" taskset -c "$CPUSET" "${cmd[@]}" \
	> "$log_file" 2>&1
	local rc=$?
	set -e
	local rss_line sampling generate wall
	rss_line="$(grep -oE 'PEAK_RSS=[0-9.]+' "$log_file" \| cut -d= -f2 \| tail -1 \|\| true)"
	sampling="$(grep -oE 'sampling completed, taking [0-9.]+' "$log_file" \| grep -oE '[0-9.]+' \| tail -1 \|\| true)"
	generate="$(grep -oE 'generate_image completed in [0-9.]+' "$log_file" \| grep -oE '[0-9.]+' \| tail -1 \|\| true)"
	wall="$(grep -oE 'WALL=[0-9.]+' "$log_file" \| cut -d= -f2 \| tail -1 \|\| true)"

	local status="ok" diff_mean="" diff_max=""
	if [[ $rc -ne 0 \|\| ! -s "$img" ]]; then
	status="failed(rc=$rc)"
	else
	[[ -z "$REF_IMAGE" ]] && REF_IMAGE="$img"
	read -r diff_mean diff_max < <("$PY" - "$REF_IMAGE" "$img" <<'PY'
	import sys
	import numpy as np
	from PIL import Image
	a = np.asarray(Image.open(sys.argv[1]).convert("RGB")).astype(int)
	b = np.asarray(Image.open(sys.argv[2]).convert("RGB")).astype(int)
	d = np.abs(a - b)
	print(f"{d.mean():.4f} {d.max()}")
	PY
	)
	fi
	echo "$name,$variant,$mlabel,$envspec,$rep,$status,$sampling,$generate,$wall,$rss_line,$diff_mean,$diff_max,images/$tag.png" >> "$CSV"
	log " $status sampling=${sampling:-?}s generate=${generate:-?}s rss=${rss_line:-?}GiB diff_mean=${diff_mean:-n/a}"
	done
	}

	# ---- sd.cpp matrix
	for row in "${MATRIX[@]}"; do
	IFS='\|' read -r name variant mkey envspec extra_args <<< "$row"
	if [[ -n "$ONLY" && "$name" != "$ONLY" ]]; then continue; fi

	sdbin="$BIN/$variant/sd-cli"
	model="$(model_file "$mkey")"
	if [[ ! -x "$sdbin" \|\| ! -f "$model" ]]; then
	log "skip $name (missing binary or model)"
	echo "$name,$variant,$mkey,$envspec,0,missing,,,,,,,," >> "$CSV"
	continue
	fi
	RUN_EXTRA="${extra_args:-}"
	run_case "$name" "$variant" "$mkey" "$envspec" \
	"$sdbin" \
	--diffusion-model "$model" \
	--vae "$MODELS/ae.safetensors" \
	--llm "$MODELS/Qwen3-4B-Instruct-2507-Q4_K_M.gguf" \
	-p "$PROMPT" -n "$NEGATIVE" \
	-W "$WIDTH" -H "$HEIGHT" --steps "$STEPS" --cfg-scale 1.0 \
	--seed "$SEED" --sampling-method euler --scheduler simple \
	-t "$THREADS" -o "__IMG__"
	done

	# ---- OpenVINO INT4 (HelloSun/Z-Image-Turbo-OpenVINO-INT4), 8 steps to match sd.cpp; same prompt/seed/size
	for ov_row in "ov-int4-8step\|8"; do
	IFS='\|' read -r name steps <<< "$ov_row"
	if [[ -n "$ONLY" && "$name" != "$ONLY" ]]; then continue; fi
	if [[ ! -x "$OV_PY" \|\| ! -d "$OV_DIR" ]]; then
	log "skip $name (missing OpenVINO venv or model; see build.sh)"
	echo "$name,openvino,OV-INT4,-,0,missing,,,,,,,," >> "$CSV"
	continue
	fi
	run_case "$name" "openvino" "OV-INT4" "-" \
	"$OV_PY" "$ROOT/tools/ov_bench.py" \
	--model-dir "$OV_DIR" -p "$PROMPT" \
	-W "$WIDTH" -H "$HEIGHT" --steps "$steps" --seed "$SEED" -o "__IMG__"
	done

	# ------------------------------------------------------------ report (medians)
	"$PY" - "$CSV" "$OUT/results.md" <<'PY'
	import csv, statistics, sys
	from collections import OrderedDict

	rows = list(csv.DictReader(open(sys.argv[1])))
	groups = OrderedDict()
	for r in rows:
	if r["status"] != "ok":
	continue
	groups.setdefault(r["name"], []).append(r)

	def med(values):
	vals = [float(v) for v in values if v not in ("", None)]
	return statistics.median(vals) if vals else None

	ref = None
	if "stock-Q4_K" in groups:
	ref = med([r["generate_s"] for r in groups["stock-Q4_K"]])

	lines = [
	"\| # \| 版本 (variant) \| 模型 \| 額外環境變數 \| 採樣 (s) \| 總生成 (s) \| 相對 stock \| 峰值 RSS (GiB) \| 與 stock 像素差 (mean/max) \|",
	"\|---\|---\|---\|---\|---\|---\|---\|---\|---\|",
	]
	for i, (name, rs) in enumerate(groups.items(), 1):
	samp = med([r["sampling_s"] for r in rs])
	gen = med([r["generate_s"] for r in rs])
	rss = med([r["peak_rss_gib"] for r in rs])
	dm = med([r["diff_mean_vs_ref"] for r in rs])
	dmx = med([r["diff_max_vs_ref"] for r in rs])
	f1 = lambda v: "n/a" if v is None else f"{v:.1f}"
	f2 = lambda v: "n/a" if v is None else f"{v:.2f}"
	speed = f"{ref / gen:.2f}x" if ref and gen else "n/a"
	diff = "ref" if name == "stock-Q4_K" else (f"{dm:.2f} / {dmx:.0f}" if dm is not None else "n/a")
	env = rs[0]["env"] if rs[0]["env"] != "-" else "-"
	if "convdirect" in name:
	env = (env + " + conv-direct") if env != "-" else "conv-direct"
	lines.append(f"\| {i} \| {rs[0]['variant']} \| {rs[0]['model']} \| {env} \| {f1(samp)} \| {f1(gen)} \| {speed} \| {f2(rss)} \| {diff} \|")
	lines.append("")
	lines.append(f"n = {max((len(v) for v in groups.values()), default=0)} run(s) per row (median). Speedup = stock-Q4_K generate time / row generate time.")
	open(sys.argv[2], "w").write("\n".join(lines) + "\n")
	print("\n".join(lines))
	PY

	log "results: $CSV"
	log "table: $OUT/results.md"