Hermes Bot commited on
fix: pass model at chat_completion call time, not constructor (HF router fix)
Browse files
shared/inference_client.py
CHANGED
|
@@ -123,7 +123,7 @@ class InferenceResult:
|
|
| 123 |
def _get_client():
|
| 124 |
"""Lazy-load the InferenceClient to keep boot fast."""
|
| 125 |
from huggingface_hub import InferenceClient
|
| 126 |
-
kwargs = {"
|
| 127 |
if INFERENCE_PROVIDER:
|
| 128 |
kwargs["provider"] = INFERENCE_PROVIDER
|
| 129 |
return InferenceClient(**kwargs)
|
|
@@ -153,6 +153,7 @@ def generate(
|
|
| 153 |
client = _get_client()
|
| 154 |
start = time.time()
|
| 155 |
response = client.chat_completion(
|
|
|
|
| 156 |
messages=messages,
|
| 157 |
max_tokens=max_new_tokens,
|
| 158 |
temperature=temperature,
|
|
|
|
| 123 |
def _get_client():
|
| 124 |
"""Lazy-load the InferenceClient to keep boot fast."""
|
| 125 |
from huggingface_hub import InferenceClient
|
| 126 |
+
kwargs = {"token": HF_TOKEN}
|
| 127 |
if INFERENCE_PROVIDER:
|
| 128 |
kwargs["provider"] = INFERENCE_PROVIDER
|
| 129 |
return InferenceClient(**kwargs)
|
|
|
|
| 153 |
client = _get_client()
|
| 154 |
start = time.time()
|
| 155 |
response = client.chat_completion(
|
| 156 |
+
model=INFERENCE_MODEL,
|
| 157 |
messages=messages,
|
| 158 |
max_tokens=max_new_tokens,
|
| 159 |
temperature=temperature,
|