scripts/start_server.sh

#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$ROOT"
[[ "$(uname -s)" == "Darwin" ]] || { echo "ERROR: MLX requires macOS." >&2; exit 1; }
[[ "$(uname -m)" == "arm64" ]] || { echo "ERROR: MLX requires Apple Silicon arm64." >&2; exit 1; }
[[ -f .venv/bin/activate ]] || { echo "ERROR: Run ./scripts/install.sh first." >&2; exit 1; }
source .venv/bin/activate
MODEL_DIR="${MODEL_DIR:-$ROOT}"
HOST="${HOST:-127.0.0.1}"
PORT="${PORT:-18190}"
MAX_KV_SIZE="${MAX_KV_SIZE:-16384}"
KV_BITS="${KV_BITS:-8}"
TOP_LOGPROBS_K="${TOP_LOGPROBS_K:-20}"
[[ -f "$MODEL_DIR/model.safetensors" ]] || { echo "ERROR: model.safetensors missing from $MODEL_DIR" >&2; exit 1; }
[[ ! -f "$MODEL_DIR/tekken.json" ]] || { echo "ERROR: tekken.json must not be present in this MLX runtime artifact." >&2; exit 1; }
echo "Model: $MODEL_DIR"
echo "Endpoint: http://$HOST:$PORT/v1"
echo "Context cap: $MAX_KV_SIZE; KV cache: ${KV_BITS}-bit"
exec mlx_vlm.server \
  --model "$MODEL_DIR" \
  --host "$HOST" \
  --port "$PORT" \
  --top-logprobs-k "$TOP_LOGPROBS_K" \
  --max-kv-size "$MAX_KV_SIZE" \
  --kv-bits "$KV_BITS" \
  --kv-group-size 64 \
  --trust-remote-code