Update README.md
Browse files
README.md
CHANGED
|
@@ -22,7 +22,7 @@ Run in a Python pipeline for local inference:
|
|
| 22 |
```python
|
| 23 |
from vllm import LLM, SamplingParams
|
| 24 |
|
| 25 |
-
model = LLM("nm-testing/OpenHermes-2.5-Mistral-7B-pruned2.4", sparsity="
|
| 26 |
prompt = "How to make banana bread?"
|
| 27 |
formatted_prompt = f"<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant"
|
| 28 |
|
|
|
|
| 22 |
```python
|
| 23 |
from vllm import LLM, SamplingParams
|
| 24 |
|
| 25 |
+
model = LLM("nm-testing/OpenHermes-2.5-Mistral-7B-pruned2.4", sparsity="semi_structured_sparse_w16a16")
|
| 26 |
prompt = "How to make banana bread?"
|
| 27 |
formatted_prompt = f"<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant"
|
| 28 |
|