Hermes Bot commited on
fix: use Llama-3.2-1B-Instruct (free HF Inference), auto-detect provider
Browse files
shared/inference_client.py
CHANGED
|
@@ -31,12 +31,12 @@ log = logging.getLogger("inference")
|
|
| 31 |
# The HF model id used for text generation (VibeThinker 1.5B, Gemma 4 12B, etc.)
|
| 32 |
INFERENCE_MODEL = os.environ.get(
|
| 33 |
"INFERENCE_MODEL",
|
| 34 |
-
"
|
| 35 |
)
|
| 36 |
|
| 37 |
# Provider: "hf-inference" (free serverless), "together", "fal-ai", "replicate"
|
| 38 |
# Free HF inference works for many small models; otherwise use a paid provider.
|
| 39 |
-
INFERENCE_PROVIDER = os.environ.get("INFERENCE_PROVIDER",
|
| 40 |
|
| 41 |
# Token — read from HF Space secrets at runtime.
|
| 42 |
HF_TOKEN = os.environ.get("HF_TOKEN") or os.environ.get("HUGGINGFACEHUB_API_TOKEN")
|
|
@@ -123,11 +123,10 @@ class InferenceResult:
|
|
| 123 |
def _get_client():
|
| 124 |
"""Lazy-load the InferenceClient to keep boot fast."""
|
| 125 |
from huggingface_hub import InferenceClient
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
)
|
| 131 |
|
| 132 |
|
| 133 |
def generate(
|
|
|
|
| 31 |
# The HF model id used for text generation (VibeThinker 1.5B, Gemma 4 12B, etc.)
|
| 32 |
INFERENCE_MODEL = os.environ.get(
|
| 33 |
"INFERENCE_MODEL",
|
| 34 |
+
"meta-llama/Llama-3.2-1B-Instruct", # 1B, free-tier, great prose
|
| 35 |
)
|
| 36 |
|
| 37 |
# Provider: "hf-inference" (free serverless), "together", "fal-ai", "replicate"
|
| 38 |
# Free HF inference works for many small models; otherwise use a paid provider.
|
| 39 |
+
INFERENCE_PROVIDER = os.environ.get("INFERENCE_PROVIDER", None)
|
| 40 |
|
| 41 |
# Token — read from HF Space secrets at runtime.
|
| 42 |
HF_TOKEN = os.environ.get("HF_TOKEN") or os.environ.get("HUGGINGFACEHUB_API_TOKEN")
|
|
|
|
| 123 |
def _get_client():
|
| 124 |
"""Lazy-load the InferenceClient to keep boot fast."""
|
| 125 |
from huggingface_hub import InferenceClient
|
| 126 |
+
kwargs = {"model": INFERENCE_MODEL, "token": HF_TOKEN}
|
| 127 |
+
if INFERENCE_PROVIDER:
|
| 128 |
+
kwargs["provider"] = INFERENCE_PROVIDER
|
| 129 |
+
return InferenceClient(**kwargs)
|
|
|
|
| 130 |
|
| 131 |
|
| 132 |
def generate(
|