Switch model source to gdubicki/Qwen3-Coder-Next-NVFP4-GB10
Browse files- README.md +7 -7
- docker-run.sh +1 -1
- start-qwen3-coder-next.sh +1 -1
README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
-
#
|
| 2 |
|
| 3 |
-
Runs [`
|
| 4 |
Tested on DGX Spark (GB10 Blackwell, SM12.1, 128 GB unified LPDDR5X).
|
| 5 |
|
| 6 |
## Model overview
|
|
@@ -51,7 +51,7 @@ Qwen3-Coder-Next is 80B total but 3B active — same throughput as Nemotron-3-Na
|
|
| 51 |
## Quick start
|
| 52 |
|
| 53 |
```bash
|
| 54 |
-
# Required — model is gated on Hugging Face (accept license at
|
| 55 |
export HF_TOKEN=hf_xxxx
|
| 56 |
|
| 57 |
bash start-qwen3-coder-next.sh
|
|
@@ -78,12 +78,12 @@ Reasoning is **ON by default**. Toggle per request:
|
|
| 78 |
# Reasoning OFF
|
| 79 |
curl -s -X POST http://localhost:8000/v1/chat/completions \
|
| 80 |
-H "Content-Type: application/json" \
|
| 81 |
-
-d '{"model":"
|
| 82 |
|
| 83 |
# Reasoning ON (default)
|
| 84 |
curl -s -X POST http://localhost:8000/v1/chat/completions \
|
| 85 |
-H "Content-Type: application/json" \
|
| 86 |
-
-d '{"model":"
|
| 87 |
```
|
| 88 |
|
| 89 |
## Cline configuration
|
|
@@ -95,7 +95,7 @@ curl -s -X POST http://localhost:8000/v1/chat/completions \
|
|
| 95 |
|-------|-------|
|
| 96 |
| Provider | OpenAI Compatible |
|
| 97 |
| Base URL | `http://<spark-ip>:8000/v1` |
|
| 98 |
-
| Model ID | `
|
| 99 |
| API Key | `dummy` (any non-empty string) |
|
| 100 |
|
| 101 |
## Files
|
|
@@ -147,6 +147,6 @@ vLLM auto-detects quantization from `quantization_config` in `config.json`; no `
|
|
| 147 |
|
| 148 |
- Docker with `nvidia-container-toolkit`
|
| 149 |
- Image: `vllm/vllm-openai:cu130-nightly`
|
| 150 |
-
- HF token with access to `
|
| 151 |
- Model weights cached locally (auto-downloaded on first run, ~45 GB):
|
| 152 |
`~/.cache/huggingface/hub/models--saricles--Qwen3-Coder-Next-NVFP4-GB10/`
|
|
|
|
| 1 |
+
# gdubicki/Qwen3-Coder-Next-NVFP4-GB10 on DGX Spark (GB10)
|
| 2 |
|
| 3 |
+
Runs [`gdubicki/Qwen3-Coder-Next-NVFP4-GB10`](https://huggingface.co/gdubicki/Qwen3-Coder-Next-NVFP4-GB10) (quantized by [saricles](https://huggingface.co/saricles/Qwen3-Coder-Next-NVFP4-GB10)) via vLLM with an OpenAI-compatible API endpoint.
|
| 4 |
Tested on DGX Spark (GB10 Blackwell, SM12.1, 128 GB unified LPDDR5X).
|
| 5 |
|
| 6 |
## Model overview
|
|
|
|
| 51 |
## Quick start
|
| 52 |
|
| 53 |
```bash
|
| 54 |
+
# Required — model is gated on Hugging Face (accept license at gdubicki/Qwen3-Coder-Next-NVFP4-GB10 first):
|
| 55 |
export HF_TOKEN=hf_xxxx
|
| 56 |
|
| 57 |
bash start-qwen3-coder-next.sh
|
|
|
|
| 78 |
# Reasoning OFF
|
| 79 |
curl -s -X POST http://localhost:8000/v1/chat/completions \
|
| 80 |
-H "Content-Type: application/json" \
|
| 81 |
+
-d '{"model":"gdubicki/Qwen3-Coder-Next-NVFP4-GB10","messages":[{"role":"user","content":"What is the capital of France?"}],"max_tokens":60,"chat_template_kwargs":{"enable_thinking":false}}'
|
| 82 |
|
| 83 |
# Reasoning ON (default)
|
| 84 |
curl -s -X POST http://localhost:8000/v1/chat/completions \
|
| 85 |
-H "Content-Type: application/json" \
|
| 86 |
+
-d '{"model":"gdubicki/Qwen3-Coder-Next-NVFP4-GB10","messages":[{"role":"user","content":"Write a binary search implementation in Python."}],"max_tokens":2000,"chat_template_kwargs":{"enable_thinking":true}}'
|
| 87 |
```
|
| 88 |
|
| 89 |
## Cline configuration
|
|
|
|
| 95 |
|-------|-------|
|
| 96 |
| Provider | OpenAI Compatible |
|
| 97 |
| Base URL | `http://<spark-ip>:8000/v1` |
|
| 98 |
+
| Model ID | `gdubicki/Qwen3-Coder-Next-NVFP4-GB10` |
|
| 99 |
| API Key | `dummy` (any non-empty string) |
|
| 100 |
|
| 101 |
## Files
|
|
|
|
| 147 |
|
| 148 |
- Docker with `nvidia-container-toolkit`
|
| 149 |
- Image: `vllm/vllm-openai:cu130-nightly`
|
| 150 |
+
- HF token with access to `gdubicki/Qwen3-Coder-Next-NVFP4-GB10` (gated — accept license first)
|
| 151 |
- Model weights cached locally (auto-downloaded on first run, ~45 GB):
|
| 152 |
`~/.cache/huggingface/hub/models--saricles--Qwen3-Coder-Next-NVFP4-GB10/`
|
docker-run.sh
CHANGED
|
@@ -27,7 +27,7 @@ docker run \
|
|
| 27 |
-e VLLM_MARLIN_USE_ATOMIC_ADD=1 \
|
| 28 |
${HF_TOKEN:+-e HF_TOKEN="${HF_TOKEN}"} \
|
| 29 |
vllm/vllm-openai:cu130-nightly \
|
| 30 |
-
|
| 31 |
--dtype auto \
|
| 32 |
--gpu-memory-utilization 0.90 \
|
| 33 |
--kv-cache-dtype fp8 \
|
|
|
|
| 27 |
-e VLLM_MARLIN_USE_ATOMIC_ADD=1 \
|
| 28 |
${HF_TOKEN:+-e HF_TOKEN="${HF_TOKEN}"} \
|
| 29 |
vllm/vllm-openai:cu130-nightly \
|
| 30 |
+
gdubicki/Qwen3-Coder-Next-NVFP4-GB10 \
|
| 31 |
--dtype auto \
|
| 32 |
--gpu-memory-utilization 0.90 \
|
| 33 |
--kv-cache-dtype fp8 \
|
start-qwen3-coder-next.sh
CHANGED
|
@@ -18,7 +18,7 @@ set -euo pipefail
|
|
| 18 |
# ---------------------------------------------------------------------------
|
| 19 |
CONTAINER_NAME="qwen3-coder-next-vllm"
|
| 20 |
IMAGE="vllm/vllm-openai:cu130-nightly"
|
| 21 |
-
MODEL="
|
| 22 |
PORT=8000
|
| 23 |
MAX_MODEL_LEN="${MAX_MODEL_LEN:-262144}"
|
| 24 |
|
|
|
|
| 18 |
# ---------------------------------------------------------------------------
|
| 19 |
CONTAINER_NAME="qwen3-coder-next-vllm"
|
| 20 |
IMAGE="vllm/vllm-openai:cu130-nightly"
|
| 21 |
+
MODEL="gdubicki/Qwen3-Coder-Next-NVFP4-GB10"
|
| 22 |
PORT=8000
|
| 23 |
MAX_MODEL_LEN="${MAX_MODEL_LEN:-262144}"
|
| 24 |
|