Update README.md
Browse files
README.md
CHANGED
|
@@ -86,7 +86,7 @@ This model can be deployed efficiently using the [vLLM](https://docs.vllm.ai/en/
|
|
| 86 |
from vllm import LLM, SamplingParams
|
| 87 |
from transformers import AutoProcessor
|
| 88 |
|
| 89 |
-
model_id = "RedHatAI/Mistral-Small-3.1-24B-Instruct-2503-
|
| 90 |
number_gpus = 1
|
| 91 |
|
| 92 |
sampling_params = SamplingParams(temperature=0.7, top_p=0.8, max_tokens=256)
|
|
|
|
| 86 |
from vllm import LLM, SamplingParams
|
| 87 |
from transformers import AutoProcessor
|
| 88 |
|
| 89 |
+
model_id = "RedHatAI/Mistral-Small-3.1-24B-Instruct-2503-quantized.w8a8"
|
| 90 |
number_gpus = 1
|
| 91 |
|
| 92 |
sampling_params = SamplingParams(temperature=0.7, top_p=0.8, max_tokens=256)
|