File size: 384 Bytes
f8d24c7 d4e1b97 f8d24c7 d4e1b97 f8d24c7 d4e1b97 f8d24c7 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 |
TYPES=(
Q8_0
Q4_K_M
IQ4_XS
IQ3_S
)
mkdir -p quantized
for T in "${TYPES[@]}"; do
echo "$T: Quantization"
if [[ $T == IQ* ]]; then
llama-quantize \
--imatrix llama-3.2.imatrix \
llama-3.2-f16.gguf \
quantized/llama-3.2-$T.gguf \
$T
else
llama-quantize \
llama-3.2-f16.gguf \
quantized/llama-3.2-$T.gguf \
$T
fi
done
|