scripts/start_llama_server.sh

#!/usr/bin/env bash
# Starts llama-server for qwen3-4b inference.
#
# Model preparation (run once):
#
# Option A — Merge LoRA into base model, then convert (simpler):
#   python3 -c "
#   from transformers import AutoModelForCausalLM, AutoTokenizer
#   from peft import PeftModel
#   base = 'new_reference/models/Qwen--Qwen3-4B-Instruct-2507'
#   adapter = 'new_reference/models/qwen3-4b-instruct-2507-emoji-sft/checkpoint-2532'
#   merged = 'new_reference/models/qwen3-4b-merged'
#   m = PeftModel.from_pretrained(AutoModelForCausalLM.from_pretrained(base, torch_dtype='auto'), adapter)
#   m.merge_and_unload().save_pretrained(merged)
#   AutoTokenizer.from_pretrained(base).save_pretrained(merged)
#   "
#   python3 /opt/homebrew/bin/convert_hf_to_gguf.py \
#     new_reference/models/qwen3-4b-merged \
#     --outfile new_reference/models/qwen3-4b-f16.gguf --outtype f16
#   /opt/homebrew/bin/llama-quantize \
#     new_reference/models/qwen3-4b-f16.gguf \
#     new_reference/models/qwen3-4b-q4_k_m.gguf Q4_K_M
#   # Then use MODEL_GGUF below (no --lora flag needed)
#
# Option B — Keep LoRA separate (requires convert_lora_to_gguf.py from llama.cpp repo):
#   python3 /opt/homebrew/bin/convert_hf_to_gguf.py \
#     new_reference/models/Qwen--Qwen3-4B-Instruct-2507 \
#     --outfile new_reference/models/qwen3-4b-base-f16.gguf --outtype f16
#   /opt/homebrew/bin/llama-quantize \
#     new_reference/models/qwen3-4b-base-f16.gguf \
#     new_reference/models/qwen3-4b-base-q4_k_m.gguf Q4_K_M
#   python3 convert_lora_to_gguf.py \
#     new_reference/models/qwen3-4b-instruct-2507-emoji-sft/checkpoint-2532 \
#     --base new_reference/models/Qwen--Qwen3-4B-Instruct-2507 \
#     --outfile new_reference/models/qwen3-4b-lora.gguf --outtype f16
#   # Then use MODEL_GGUF + LORA_GGUF below

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
ROOT_DIR="$(dirname "$SCRIPT_DIR")"

# Option A (merged model, no adapter):
MODEL_GGUF="$ROOT_DIR/new_reference/models/qwen3-4b-q4_k_m.gguf"

# Option B (base model + separate adapter — uncomment and set LORA_GGUF):
# MODEL_GGUF="$ROOT_DIR/new_reference/models/qwen3-4b-base-q4_k_m.gguf"
# LORA_GGUF="$ROOT_DIR/new_reference/models/qwen3-4b-lora.gguf"

LLAMA_SERVER="/opt/homebrew/bin/llama-server"

exec "$LLAMA_SERVER" \
  --model "$MODEL_GGUF" \
  ${LORA_GGUF:+--lora "$LORA_GGUF"} \
  --n-gpu-layers -1 \
  --parallel 8 \
  --ctx-size 8192 \
  --port 8081 \
  --host 127.0.0.1 \
  --no-webui