scripts/quantizer.sh

TYPES=(
  Q8_0
  Q4_K_M
  IQ4_XS
  IQ3_S
)

mkdir -p quantized

for T in "${TYPES[@]}"; do
  echo "$T: Quantization"

  if [[ $T == IQ* ]]; then
    llama-quantize \
      --imatrix llama-3.2.imatrix \
      llama-3.2-f16.gguf \
      quantized/llama-3.2-$T.gguf \
      $T
  else
    llama-quantize \
      llama-3.2-f16.gguf \
      quantized/llama-3.2-$T.gguf \
      $T
  fi
done