File size: 4,245 Bytes
9a5e3ef 0fdbfb9 9a5e3ef 0fdbfb9 9a5e3ef 0fdbfb9 9a5e3ef 0fdbfb9 0a487d5 9a5e3ef 0fdbfb9 0a487d5 0fdbfb9 0a487d5 0fdbfb9 0a487d5 0fdbfb9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 | #!/usr/bin/env bash
# Pull official Qwen3-Coder-Next-FP8 (40 shards). Fail-closed SHA bind when SUMS present.
# DAY-0: FP8 quality path only — not NVFP4 speed claim. diy_gguf=false.
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
REPO="${QWEN_FP8_REPO:-Qwen/Qwen3-Coder-Next-FP8}"
REV="${QWEN_FP8_REV:-da6e2ed27304dd39abadd9c82ef50e8de67bdd4c}"
DEST="${QWEN_FP8_DIR:-$HOME/models/qwen3-coder-next/Qwen3-Coder-Next-FP8}"
# Pack receipt takes priority; allow override
SUMS="${QWEN_FP8_SHA256SUMS:-}"
if [[ -z "${SUMS}" ]]; then
if [[ -f "${ROOT}/results/SHA256SUMS.fp8" ]]; then
SUMS="${ROOT}/results/SHA256SUMS.fp8"
elif [[ -f "${ROOT}/SHA256SUMS.fp8" ]]; then
SUMS="${ROOT}/SHA256SUMS.fp8"
fi
fi
EXPECTED_SHARDS="${EXPECTED_SHARDS:-40}"
REQUIRE_SUMS="${REQUIRE_SUMS:-1}"
HF_HUB_ENABLE_HF_TRANSFER="${HF_HUB_ENABLE_HF_TRANSFER:-1}"
export HF_HUB_ENABLE_HF_TRANSFER
if [[ "${REQUIRE_SUMS}" == "1" && -z "${SUMS:-}" ]]; then
echo "FAIL-CLOSED: no SHA256SUMS.fp8 — set QWEN_FP8_SHA256SUMS or ship results/SHA256SUMS.fp8" >&2
exit 2
fi
if [[ -n "${SUMS:-}" && ! -f "${SUMS}" ]]; then
echo "FAIL-CLOSED: SUMS file missing: ${SUMS}" >&2
exit 2
fi
mkdir -p "${DEST}"
echo "[pull-fp8] repo=${REPO} rev=${REV} dest=${DEST}"
echo "[pull-fp8] sums=${SUMS:-none} expected_shards=${EXPECTED_SHARDS}"
if command -v huggingface-cli >/dev/null 2>&1; then
huggingface-cli download "${REPO}" \
--revision "${REV}" \
--local-dir "${DEST}" \
--local-dir-use-symlinks False
elif command -v hf >/dev/null 2>&1; then
hf download "${REPO}" \
--revision "${REV}" \
--local-dir "${DEST}"
else
echo "FAIL-CLOSED: need huggingface-cli or hf" >&2
exit 2
fi
# config presence
if [[ ! -f "${DEST}/config.json" ]]; then
echo "FAIL-CLOSED: missing config.json under ${DEST}" >&2
exit 2
fi
# shard count
mapfile -t SHARDS < <(find "${DEST}" -maxdepth 1 -type f -name 'model-*-of-*.safetensors' | sort)
sc="${#SHARDS[@]}"
if [[ "${sc}" -lt "${EXPECTED_SHARDS}" ]]; then
echo "FAIL-CLOSED: shard count ${sc} < expected ${EXPECTED_SHARDS}" >&2
exit 3
fi
echo "[pull-fp8] shards_found=${sc}"
# SHA bind (all files listed in SUMS)
if [[ -n "${SUMS:-}" && -f "${SUMS}" ]]; then
fail=0
checked=0
while read -r hash name; do
[[ -z "${hash:-}" || "${hash}" =~ ^# ]] && continue
# support "hash name" GNU form
if [[ -z "${name:-}" ]]; then
continue
fi
# skip non-hex
if [[ ! "${hash}" =~ ^[0-9a-fA-F]{64}$ ]]; then
continue
fi
f="${DEST}/${name}"
if [[ ! -f "${f}" ]]; then
# allow name-only basename match
f2="$(find "${DEST}" -maxdepth 2 -type f -name "${name}" | head -1 || true)"
if [[ -z "${f2}" ]]; then
echo "FAIL-CLOSED: listed file missing: ${name}" >&2
fail=1
continue
fi
f="${f2}"
fi
got="$(sha256sum "${f}" | awk '{print $1}')"
checked=$((checked+1))
if [[ "${got}" != "${hash}" ]]; then
echo "FAIL-CLOSED: sha256 mismatch ${name}" >&2
echo " expected ${hash}" >&2
echo " got ${got}" >&2
fail=1
else
echo "OK ${name}"
fi
done < <(awk 'NF>=2 && $1 ~ /^[0-9a-fA-F]{64}$/ {print $1, $NF}' "${SUMS}")
if [[ "${checked}" -lt 1 ]]; then
echo "FAIL-CLOSED: SUMS produced zero hash rows: ${SUMS}" >&2
exit 2
fi
# enforce 40 model shards specifically
shard_rows="$(awk 'NF>=2 && $1 ~ /^[0-9a-fA-F]{64}$/ && $NF ~ /model-.*\\.safetensors$/ {c++} END{print c+0}' "${SUMS}")"
if [[ "${shard_rows}" -lt "${EXPECTED_SHARDS}" ]]; then
echo "FAIL-CLOSED: SUMS lists ${shard_rows} model shards < ${EXPECTED_SHARDS}" >&2
exit 2
fi
if [[ "${fail}" -ne 0 ]]; then
echo "FAIL-CLOSED: one or more SHA mismatches — not writing PULL_COMPLETE" >&2
exit 3
fi
echo "[pull-fp8] sha_checked=${checked} shard_rows=${shard_rows}"
fi
MARKER="${DEST}/PULL_COMPLETE"
{
echo "repo=${REPO}"
echo "rev=${REV}"
echo "dest=${DEST}"
echo "shards=${sc}"
echo "sums=${SUMS:-none}"
echo "status=pull_complete"
echo "day0=fp8_quality_not_nvfp4"
echo "diy_gguf=false"
echo "written_at=$(date -Iseconds 2>/dev/null || date)"
} | tee "${MARKER}"
echo "OK FP8 pull complete · marker=${MARKER}"
|