llama.cpp/examples/sycl/win-start-svr.bat

::  MIT license
::  Copyright (C) 2024 Intel Corporation
::  SPDX-License-Identifier: MIT

@echo off
setlocal EnableExtensions EnableDelayedExpansion

set "BIN_FILE=.\build\bin\llama-server.exe"
set "SEED=0"
set "GPUS_SETTING="

set "MODEL_FILE=..\models\Qwen3.5-4B-Q4_0.gguf"
set "NGL=99"
set "CONTEXT=4096"
set "GGML_SYCL_DEVICE=-1"
set "SPLIT_MODE=layer"
set "LOG_VERBOSE=3"

if "%~1"=="" goto after_args

:parse_args
if "%~1"=="" goto after_args

if /I "%~1"=="-c" (
  if "%~2"=="" goto missing_value
  set "CONTEXT=%~2"
  shift
  shift
  goto parse_args
)
if /I "%~1"=="--context" (
  if "%~2"=="" goto missing_value
  set "CONTEXT=%~2"
  shift
  shift
  goto parse_args
)

if /I "%~1"=="-m" (
  if "%~2"=="" goto missing_value
  set "MODEL_FILE=%~2"
  shift
  shift
  goto parse_args
)
if /I "%~1"=="--model" (
  if "%~2"=="" goto missing_value
  set "MODEL_FILE=%~2"
  shift
  shift
  goto parse_args
)

if /I "%~1"=="-mg" (
  if "%~2"=="" goto missing_value
  set "GGML_SYCL_DEVICE=%~2"
  set "SPLIT_MODE=none"
  shift
  shift
  goto parse_args
)
if /I "%~1"=="--main-gpu" (
  if "%~2"=="" goto missing_value
  set "GGML_SYCL_DEVICE=%~2"
  set "SPLIT_MODE=none"
  shift
  shift
  goto parse_args
)

if /I "%~1"=="-sm" (
  if "%~2"=="" goto missing_value
  set "SPLIT_MODE=%~2"
  shift
  shift
  goto parse_args
)
if /I "%~1"=="--split-mode" (
  if "%~2"=="" goto missing_value
  set "SPLIT_MODE=%~2"
  shift
  shift
  goto parse_args
)

if /I "%~1"=="-ngl" (
  if "%~2"=="" goto missing_value
  set "NGL=%~2"
  shift
  shift
  goto parse_args
)
if /I "%~1"=="--n-gpu-layers" (
  if "%~2"=="" goto missing_value
  set "NGL=%~2"
  shift
  shift
  goto parse_args
)

if /I "%~1"=="-lv" (
  if "%~2"=="" goto missing_value
  set "LOG_VERBOSE=%~2"
  shift
  shift
  goto parse_args
)
if /I "%~1"=="--log-verbosity" (
  if "%~2"=="" goto missing_value
  set "LOG_VERBOSE=%~2"
  shift
  shift
  goto parse_args
)

if /I "%~1"=="-h" goto help
if /I "%~1"=="--help" goto help

echo Invalid option: %~1
exit /b 1

:missing_value
echo Missing value for option: %~1
exit /b 1

:help
echo Usage: %~n0 [OPTIONS]
echo.
echo This script processes files with specified options.
echo.
echo Options:
echo   -h, --help    Display this help message and exit.
echo   -c, --context ^<value^>    Set context length. Bigger need more memory.
echo   -m, --model   ^<value^>    Full model file path.
echo   -mg,--main-gpu ^<value^>   Set main GPU ID (0 - n) for single GPU mode.
echo   -sm,--split-mode ^<value^> How to split the model across multiple GPUs, one of:
echo                             - none: use one GPU only
echo                             - layer (default): split layers and KV across GPUs
echo                             - row: split rows across GPUs
echo   -ngl,--n-gpu-layers ^<value^>  Max. number of layers to store in VRAM (default: -1)
echo   -lv,--log-verbosity ^<value^>  Set the verbosity threshold. Messages with a higher verbosity will be
echo                                ignored. Values:
echo                                 - 0: generic output
echo                                 - 1: error
echo                                 - 2: warning
echo                                 - 3: info
echo                                 - 4: debug
exit /b 0

:after_args

REM In Windows CMD, source is not available; call oneAPI setvars if present.
if exist "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" (
  call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" >nul
) else (
  echo Warning: oneAPI setvars.bat not found. Continuing without environment setup.
)

REM Support malloc device memory more than 4GB.
set "UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS=1"
echo UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS=%UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS%

if not "%GGML_SYCL_DEVICE%"=="-1" (
  echo Use %GGML_SYCL_DEVICE% as main GPU
  REM Use single GPU only.
  set "GPUS_SETTING=-mg %GGML_SYCL_DEVICE% -sm %SPLIT_MODE%"
  echo ONEAPI_DEVICE_SELECTOR=%ONEAPI_DEVICE_SELECTOR%
) else (
  echo Use all Intel GPUs, including iGPU ^& dGPU
  set "GPUS_SETTING=-sm %SPLIT_MODE%"
)

echo run cmd: ZES_ENABLE_SYSMAN=1 %BIN_FILE% -m "%MODEL_FILE%" -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --mmap --host 0.0.0.0 --port 8000
set "ZES_ENABLE_SYSMAN=1"
%BIN_FILE% -m "%MODEL_FILE%" -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --mmap --host 0.0.0.0 --port 8000

endlocal