scripts/pull_official_fp8.sh

#!/usr/bin/env bash
# Pull official Qwen3-Coder-Next-FP8 (40 shards). Fail-closed SHA bind when SUMS present.
# DAY-0: FP8 quality path only — not NVFP4 speed claim. diy_gguf=false.
set -euo pipefail

ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
REPO="${QWEN_FP8_REPO:-Qwen/Qwen3-Coder-Next-FP8}"
REV="${QWEN_FP8_REV:-da6e2ed27304dd39abadd9c82ef50e8de67bdd4c}"
DEST="${QWEN_FP8_DIR:-$HOME/models/qwen3-coder-next/Qwen3-Coder-Next-FP8}"
# Pack receipt takes priority; allow override
SUMS="${QWEN_FP8_SHA256SUMS:-}"
if [[ -z "${SUMS}" ]]; then
  if [[ -f "${ROOT}/results/SHA256SUMS.fp8" ]]; then
    SUMS="${ROOT}/results/SHA256SUMS.fp8"
  elif [[ -f "${ROOT}/SHA256SUMS.fp8" ]]; then
    SUMS="${ROOT}/SHA256SUMS.fp8"
  fi
fi
EXPECTED_SHARDS="${EXPECTED_SHARDS:-40}"
REQUIRE_SUMS="${REQUIRE_SUMS:-1}"
HF_HUB_ENABLE_HF_TRANSFER="${HF_HUB_ENABLE_HF_TRANSFER:-1}"
export HF_HUB_ENABLE_HF_TRANSFER

if [[ "${REQUIRE_SUMS}" == "1" && -z "${SUMS:-}" ]]; then
  echo "FAIL-CLOSED: no SHA256SUMS.fp8 — set QWEN_FP8_SHA256SUMS or ship results/SHA256SUMS.fp8" >&2
  exit 2
fi
if [[ -n "${SUMS:-}" && ! -f "${SUMS}" ]]; then
  echo "FAIL-CLOSED: SUMS file missing: ${SUMS}" >&2
  exit 2
fi

mkdir -p "${DEST}"
echo "[pull-fp8] repo=${REPO} rev=${REV} dest=${DEST}"
echo "[pull-fp8] sums=${SUMS:-none} expected_shards=${EXPECTED_SHARDS}"

if command -v huggingface-cli >/dev/null 2>&1; then
  huggingface-cli download "${REPO}" \
    --revision "${REV}" \
    --local-dir "${DEST}" \
    --local-dir-use-symlinks False
elif command -v hf >/dev/null 2>&1; then
  hf download "${REPO}" \
    --revision "${REV}" \
    --local-dir "${DEST}"
else
  echo "FAIL-CLOSED: need huggingface-cli or hf" >&2
  exit 2
fi

# config presence
if [[ ! -f "${DEST}/config.json" ]]; then
  echo "FAIL-CLOSED: missing config.json under ${DEST}" >&2
  exit 2
fi

# shard count
mapfile -t SHARDS < <(find "${DEST}" -maxdepth 1 -type f -name 'model-*-of-*.safetensors' | sort)
sc="${#SHARDS[@]}"
if [[ "${sc}" -lt "${EXPECTED_SHARDS}" ]]; then
  echo "FAIL-CLOSED: shard count ${sc} < expected ${EXPECTED_SHARDS}" >&2
  exit 3
fi
echo "[pull-fp8] shards_found=${sc}"

# SHA bind (all files listed in SUMS)
if [[ -n "${SUMS:-}" && -f "${SUMS}" ]]; then
  fail=0
  checked=0
  while read -r hash name; do
    [[ -z "${hash:-}" || "${hash}" =~ ^# ]] && continue
    # support "hash  name" GNU form
    if [[ -z "${name:-}" ]]; then
      continue
    fi
    # skip non-hex
    if [[ ! "${hash}" =~ ^[0-9a-fA-F]{64}$ ]]; then
      continue
    fi
    f="${DEST}/${name}"
    if [[ ! -f "${f}" ]]; then
      # allow name-only basename match
      f2="$(find "${DEST}" -maxdepth 2 -type f -name "${name}" | head -1 || true)"
      if [[ -z "${f2}" ]]; then
        echo "FAIL-CLOSED: listed file missing: ${name}" >&2
        fail=1
        continue
      fi
      f="${f2}"
    fi
    got="$(sha256sum "${f}" | awk '{print $1}')"
    checked=$((checked+1))
    if [[ "${got}" != "${hash}" ]]; then
      echo "FAIL-CLOSED: sha256 mismatch ${name}" >&2
      echo "  expected ${hash}" >&2
      echo "  got      ${got}" >&2
      fail=1
    else
      echo "OK ${name}"
    fi
  done < <(awk 'NF>=2 && $1 ~ /^[0-9a-fA-F]{64}$/ {print $1, $NF}' "${SUMS}")
  if [[ "${checked}" -lt 1 ]]; then
    echo "FAIL-CLOSED: SUMS produced zero hash rows: ${SUMS}" >&2
    exit 2
  fi
  # enforce 40 model shards specifically
  shard_rows="$(awk 'NF>=2 && $1 ~ /^[0-9a-fA-F]{64}$/ && $NF ~ /model-.*\\.safetensors$/ {c++} END{print c+0}' "${SUMS}")"
  if [[ "${shard_rows}" -lt "${EXPECTED_SHARDS}" ]]; then
    echo "FAIL-CLOSED: SUMS lists ${shard_rows} model shards < ${EXPECTED_SHARDS}" >&2
    exit 2
  fi
  if [[ "${fail}" -ne 0 ]]; then
    echo "FAIL-CLOSED: one or more SHA mismatches — not writing PULL_COMPLETE" >&2
    exit 3
  fi
  echo "[pull-fp8] sha_checked=${checked} shard_rows=${shard_rows}"
fi

MARKER="${DEST}/PULL_COMPLETE"
{
  echo "repo=${REPO}"
  echo "rev=${REV}"
  echo "dest=${DEST}"
  echo "shards=${sc}"
  echo "sums=${SUMS:-none}"
  echo "status=pull_complete"
  echo "day0=fp8_quality_not_nvfp4"
  echo "diy_gguf=false"
  echo "written_at=$(date -Iseconds 2>/dev/null || date)"
} | tee "${MARKER}"

echo "OK FP8 pull complete · marker=${MARKER}"