File size: 566 Bytes
1db7196 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 | # cd /home/mshahidul/llama.cpp
# export CUDA_DEVICE_ORDER=PCI_BUS_ID
# export CUDA_VISIBLE_DEVICES=3
# ./build/bin/llama-server \
# -m /home/mshahidul/readctrl/models/translategemma-27b-it-Q8_0.gguf \
# -ngl 99 \
# -fa on \
# -c 16384 \
# --port 8081 \
# --host 0.0.0.0
cd /home/mshahidul/llama.cpp
export CUDA_DEVICE_ORDER=PCI_BUS_ID
export CUDA_VISIBLE_DEVICES=4
./build/bin/llama-server \
-m unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF/DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf \
-ngl 99 \
-fa on \
-c 16384 \
--port 8085 \
--host 0.0.0.0 |