File size: 1,139 Bytes
a66a9c7 290f61d a66a9c7 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 | #!/bin/bash
# Speculative-decoding A/B on the 35B weights (single stream).
# For a 3B-active MoE the verify pass reads the weights ONCE for several tokens, so this is the one remaining
# lever that cuts bytes-per-token without needing concurrent requests. Payoff depends entirely on acceptance
# rate, so this measures generated tokens/s with and without the draft, same prompt, temp 0.
cd /home/bob/OldIron/engine || exit 1
T=/mnt/flare/OldIron/weights/Qwen3.6-35B-A3B-Q4_0-otq4.gguf
D=/home/bob/OldIron/drafts/Qwen3.5-0.8B-Q4_K_M.gguf
P="Write a Python function that reverses a singly linked list, then explain the time complexity."
N=96
R=/home/bob/OldIron/results
echo "=== baseline: no draft, t=24 ==="
nice -n 19 ./bin/llama-cli -m "$T" -ngl 0 --no-op-offload -t 24 -c 1024 -n $N --temp 0 -st -p "$P" </dev/null 2>&1 \
| grep -E "^\[ Prompt:|Generation:" | tail -1
echo "=== speculative: draft Qwen3.5-0.8B Q4, t=24 ==="
nice -n 19 ./bin/llama-cli -m "$T" -md "$D" -ngl 0 --no-op-offload -t 24 -c 1024 -n $N --temp 0 -st -p "$P" </dev/null 2>&1 \
| grep -E "^\[ Prompt:|Generation:|draft|accept" | tail -4
echo "=== done ==="
|