File size: 1,668 Bytes
5828b5b 4a635b8 1481e5c 5828b5b 1481e5c 5828b5b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 | #!/usr/bin/env bash
# Start the FastAPI UI immediately (binds :7860 so the HF health check passes at
# once) and warm the model up in the background. On a cold start the frontend
# status pill reads "connecting…" until llama-server answers /v1/models — that's
# the natural "model waking up" state.
set -uo pipefail
MODEL_PATH="${MODEL_PATH:-/home/user/app/model.gguf}"
# The base image ships llama-server in /app alongside its shared libs
# (libllama-server-impl.so, libggml*.so). Its RPATH is relative to the binary, so
# we MUST run the real /app path (not a symlink) and point the loader at /app.
export LD_LIBRARY_PATH="/app:${LD_LIBRARY_PATH:-}"
LLAMA_BIN=""
for p in /app/llama-server /llama-server /usr/local/bin/llama-server /usr/bin/llama-server; do
[ -x "$p" ] && LLAMA_BIN="$p" && break
done
[ -z "$LLAMA_BIN" ] && LLAMA_BIN="$(command -v llama-server || true)"
echo "[start] using llama-server at: ${LLAMA_BIN:-NOT FOUND} (LD_LIBRARY_PATH=$LD_LIBRARY_PATH)"
serve() { # $1 = context window
echo "[start] llama-server -c $1 loading ${MODEL_PATH}"
"$LLAMA_BIN" -m "$MODEL_PATH" --host 127.0.0.1 --port 8080 \
--alias "${PIXELLOCK_MODEL:-pixellock}" \
-ngl 999 -np 1 -b 2048 -ub 512 -t 4 --jinja -c "$1"
}
# f16 KV cache, ample context, no exotic flags — the most robust cold start on a
# 16GB T4. The GBNF grammar (not flash-attn or KV quant) is what guarantees
# pixel-perfect output, so plain f16 is the safe, high-quality choice. If 12288
# can't fit, fall back to a smaller window automatically.
( serve 12288 || serve 8192 ) 2>&1 &
echo "[start] Gradio UI on :${PORT:-7860} (model warming in background)"
exec python3 app.py
|