#!/bin/bash
# Agnes PFFN-FP8 serving entrypoint, pinned to lmsysorg/sglang:v0.5.16.
#
# Usage:
#   docker run --gpus all --shm-size 64g -p 30001:30002 \
#     -v /path/to/this/model:/model \
#     lmsysorg/sglang:v0.5.16 bash /model/serve.sh
#
# Extra sglang flags may be appended after serve.sh, e.g.:
#   ... bash /model/serve.sh --tp 4
set -e
MODEL_DIR="$(cd "$(dirname "$0")" && pwd)"
SGL_PKG=/sgl-workspace/sglang/python/sglang

# Overlay the Agnes support files onto the stock sglang package
# (inside this container only; the model directory is never modified).
cp -r "$MODEL_DIR/sglang_patch/srt" "$MODEL_DIR/sglang_patch/kernels" "$SGL_PKG/"

exec sglang serve --model-path "$MODEL_DIR" --trust-remote-code --tp 8 \
  --context-length 1048576 --mem-fraction-static 0.90 \
  --host 0.0.0.0 --port 30002 "$@"