Hermes Bot commited on
Commit
bc6b501
·
unverified ·
1 Parent(s): 2922094

fix: pass model at chat_completion call time, not constructor (HF router fix)

Browse files
Files changed (1) hide show
  1. shared/inference_client.py +2 -1
shared/inference_client.py CHANGED
@@ -123,7 +123,7 @@ class InferenceResult:
123
  def _get_client():
124
  """Lazy-load the InferenceClient to keep boot fast."""
125
  from huggingface_hub import InferenceClient
126
- kwargs = {"model": INFERENCE_MODEL, "token": HF_TOKEN}
127
  if INFERENCE_PROVIDER:
128
  kwargs["provider"] = INFERENCE_PROVIDER
129
  return InferenceClient(**kwargs)
@@ -153,6 +153,7 @@ def generate(
153
  client = _get_client()
154
  start = time.time()
155
  response = client.chat_completion(
 
156
  messages=messages,
157
  max_tokens=max_new_tokens,
158
  temperature=temperature,
 
123
  def _get_client():
124
  """Lazy-load the InferenceClient to keep boot fast."""
125
  from huggingface_hub import InferenceClient
126
+ kwargs = {"token": HF_TOKEN}
127
  if INFERENCE_PROVIDER:
128
  kwargs["provider"] = INFERENCE_PROVIDER
129
  return InferenceClient(**kwargs)
 
153
  client = _get_client()
154
  start = time.time()
155
  response = client.chat_completion(
156
+ model=INFERENCE_MODEL,
157
  messages=messages,
158
  max_tokens=max_new_tokens,
159
  temperature=temperature,