HelloSun commited on
Commit
5b4c4e3
·
verified ·
1 Parent(s): 5d0617d

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +2 -2
app.py CHANGED
@@ -538,7 +538,7 @@ async def start_llama_server():
538
  "--model", MODEL_PATH,
539
  "--host", "127.0.0.1",
540
  "--port", str(LLAMA_PORT),
541
- "--mlock", # 要求鎖住模型於實體記憶體,避免被 swap 拖慢速度
542
  "--flash-attn", "on", # flash attention,加速並省顯存/記憶體
543
  "--cont-batching", # 連續批次處理,吞吐量最佳化
544
  "--ctx-size", "8192",
@@ -548,7 +548,7 @@ async def start_llama_server():
548
  "--cache-type-k", "q8_0",
549
  "--cache-type-v", "q8_0",
550
  "--parallel", "1", # MTP 投機解碼目前僅支援 --parallel 1
551
- "--spec-type", "mtp", # 這顆 GGUF 內建 MTP 頭,開啟後可明顯加速 token 生成
552
  "--spec-draft-n-max", "2",
553
  ]
554
  cmd += ["--n-gpu-layers", "99" if gpu else "0"]
 
538
  "--model", MODEL_PATH,
539
  "--host", "127.0.0.1",
540
  "--port", str(LLAMA_PORT),
541
+ "--load-mode mlock ", # 要求鎖住模型於實體記憶體,避免被 swap 拖慢速度
542
  "--flash-attn", "on", # flash attention,加速並省顯存/記憶體
543
  "--cont-batching", # 連續批次處理,吞吐量最佳化
544
  "--ctx-size", "8192",
 
548
  "--cache-type-k", "q8_0",
549
  "--cache-type-v", "q8_0",
550
  "--parallel", "1", # MTP 投機解碼目前僅支援 --parallel 1
551
+ "--spec-type", "draft-mtp", # 這顆 GGUF 內建 MTP 頭,開啟後可明顯加速 token 生成
552
  "--spec-draft-n-max", "2",
553
  ]
554
  cmd += ["--n-gpu-layers", "99" if gpu else "0"]