File size: 1,187 Bytes
4dc5bc6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 | #!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$ROOT"
[[ "$(uname -s)" == "Darwin" ]] || { echo "ERROR: MLX requires macOS." >&2; exit 1; }
[[ "$(uname -m)" == "arm64" ]] || { echo "ERROR: MLX requires Apple Silicon arm64." >&2; exit 1; }
[[ -f .venv/bin/activate ]] || { echo "ERROR: Run ./scripts/install.sh first." >&2; exit 1; }
source .venv/bin/activate
MODEL_DIR="${MODEL_DIR:-$ROOT}"
HOST="${HOST:-127.0.0.1}"
PORT="${PORT:-18190}"
MAX_KV_SIZE="${MAX_KV_SIZE:-16384}"
KV_BITS="${KV_BITS:-8}"
TOP_LOGPROBS_K="${TOP_LOGPROBS_K:-20}"
[[ -f "$MODEL_DIR/model.safetensors" ]] || { echo "ERROR: model.safetensors missing from $MODEL_DIR" >&2; exit 1; }
[[ ! -f "$MODEL_DIR/tekken.json" ]] || { echo "ERROR: tekken.json must not be present in this MLX runtime artifact." >&2; exit 1; }
echo "Model: $MODEL_DIR"
echo "Endpoint: http://$HOST:$PORT/v1"
echo "Context cap: $MAX_KV_SIZE; KV cache: ${KV_BITS}-bit"
exec mlx_vlm.server \
--model "$MODEL_DIR" \
--host "$HOST" \
--port "$PORT" \
--top-logprobs-k "$TOP_LOGPROBS_K" \
--max-kv-size "$MAX_KV_SIZE" \
--kv-bits "$KV_BITS" \
--kv-group-size 64 \
--trust-remote-code
|