Hermes Bot commited on
Commit
2922094
·
unverified ·
1 Parent(s): b1b7ff5

fix: use Llama-3.2-1B-Instruct (free HF Inference), auto-detect provider

Browse files
Files changed (1) hide show
  1. shared/inference_client.py +6 -7
shared/inference_client.py CHANGED
@@ -31,12 +31,12 @@ log = logging.getLogger("inference")
31
  # The HF model id used for text generation (VibeThinker 1.5B, Gemma 4 12B, etc.)
32
  INFERENCE_MODEL = os.environ.get(
33
  "INFERENCE_MODEL",
34
- "Qwen/Qwen2.5-1.5B-Instruct", # small, fast, free-tier friendly
35
  )
36
 
37
  # Provider: "hf-inference" (free serverless), "together", "fal-ai", "replicate"
38
  # Free HF inference works for many small models; otherwise use a paid provider.
39
- INFERENCE_PROVIDER = os.environ.get("INFERENCE_PROVIDER", "hf-inference")
40
 
41
  # Token — read from HF Space secrets at runtime.
42
  HF_TOKEN = os.environ.get("HF_TOKEN") or os.environ.get("HUGGINGFACEHUB_API_TOKEN")
@@ -123,11 +123,10 @@ class InferenceResult:
123
  def _get_client():
124
  """Lazy-load the InferenceClient to keep boot fast."""
125
  from huggingface_hub import InferenceClient
126
- return InferenceClient(
127
- model=INFERENCE_MODEL,
128
- token=HF_TOKEN,
129
- provider=INFERENCE_PROVIDER,
130
- )
131
 
132
 
133
  def generate(
 
31
  # The HF model id used for text generation (VibeThinker 1.5B, Gemma 4 12B, etc.)
32
  INFERENCE_MODEL = os.environ.get(
33
  "INFERENCE_MODEL",
34
+ "meta-llama/Llama-3.2-1B-Instruct", # 1B, free-tier, great prose
35
  )
36
 
37
  # Provider: "hf-inference" (free serverless), "together", "fal-ai", "replicate"
38
  # Free HF inference works for many small models; otherwise use a paid provider.
39
+ INFERENCE_PROVIDER = os.environ.get("INFERENCE_PROVIDER", None)
40
 
41
  # Token — read from HF Space secrets at runtime.
42
  HF_TOKEN = os.environ.get("HF_TOKEN") or os.environ.get("HUGGINGFACEHUB_API_TOKEN")
 
123
  def _get_client():
124
  """Lazy-load the InferenceClient to keep boot fast."""
125
  from huggingface_hub import InferenceClient
126
+ kwargs = {"model": INFERENCE_MODEL, "token": HF_TOKEN}
127
+ if INFERENCE_PROVIDER:
128
+ kwargs["provider"] = INFERENCE_PROVIDER
129
+ return InferenceClient(**kwargs)
 
130
 
131
 
132
  def generate(