File size: 498 Bytes
cb2428f |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 |
# gptq quantize
CUDA_VISIBLE_DEVICES=0 swift export \
--model Shanghai_AI_Laboratory/internlm2-1_8b-reward \
--output_dir output/internlm2-1_8b-reward-gptq-int4 \
--quant_bits 4 \
--quant_method gptq \
--dataset 'AI-ModelScope/alpaca-gpt4-data-zh#1000' 'AI-ModelScope/alpaca-gpt4-data-en#1000'
# infer
CUDA_VISIBLE_DEVICES=0 swift infer \
--model output/internlm2-1_8b-reward-gptq-int4 \
--val_dataset 'AI-ModelScope/alpaca-gpt4-data-zh#1000' \
--max_batch_size 16
|