Spaces:
Paused
Paused
Update app.py
Browse files
app.py
CHANGED
|
@@ -538,7 +538,7 @@ async def start_llama_server():
|
|
| 538 |
"--model", MODEL_PATH,
|
| 539 |
"--host", "127.0.0.1",
|
| 540 |
"--port", str(LLAMA_PORT),
|
| 541 |
-
"--mlock", # 要求鎖住模型於實體記憶體,避免被 swap 拖慢速度
|
| 542 |
"--flash-attn", "on", # flash attention,加速並省顯存/記憶體
|
| 543 |
"--cont-batching", # 連續批次處理,吞吐量最佳化
|
| 544 |
"--ctx-size", "8192",
|
|
@@ -548,7 +548,7 @@ async def start_llama_server():
|
|
| 548 |
"--cache-type-k", "q8_0",
|
| 549 |
"--cache-type-v", "q8_0",
|
| 550 |
"--parallel", "1", # MTP 投機解碼目前僅支援 --parallel 1
|
| 551 |
-
"--spec-type", "mtp", # 這顆 GGUF 內建 MTP 頭,開啟後可明顯加速 token 生成
|
| 552 |
"--spec-draft-n-max", "2",
|
| 553 |
]
|
| 554 |
cmd += ["--n-gpu-layers", "99" if gpu else "0"]
|
|
|
|
| 538 |
"--model", MODEL_PATH,
|
| 539 |
"--host", "127.0.0.1",
|
| 540 |
"--port", str(LLAMA_PORT),
|
| 541 |
+
"--load-mode mlock ", # 要求鎖住模型於實體記憶體,避免被 swap 拖慢速度
|
| 542 |
"--flash-attn", "on", # flash attention,加速並省顯存/記憶體
|
| 543 |
"--cont-batching", # 連續批次處理,吞吐量最佳化
|
| 544 |
"--ctx-size", "8192",
|
|
|
|
| 548 |
"--cache-type-k", "q8_0",
|
| 549 |
"--cache-type-v", "q8_0",
|
| 550 |
"--parallel", "1", # MTP 投機解碼目前僅支援 --parallel 1
|
| 551 |
+
"--spec-type", "draft-mtp", # 這顆 GGUF 內建 MTP 頭,開啟後可明顯加速 token 生成
|
| 552 |
"--spec-draft-n-max", "2",
|
| 553 |
]
|
| 554 |
cmd += ["--n-gpu-layers", "99" if gpu else "0"]
|