Spaces:
Running
Running
Enable proxy LLM mode with API_KEY and real default model
Browse files- inference.py +7 -7
- tests/test_inference_unit.py +3 -7
inference.py
CHANGED
|
@@ -14,7 +14,7 @@ API_BASE_URL
|
|
| 14 |
|
| 15 |
MODEL_NAME
|
| 16 |
Model identifier to use for LLM inference.
|
| 17 |
-
Default: ``
|
| 18 |
|
| 19 |
API_KEY
|
| 20 |
Proxy/API authentication token injected by the evaluator.
|
|
@@ -69,7 +69,7 @@ from vocabulary import (
|
|
| 69 |
# Configuration
|
| 70 |
# ---------------------------------------------------------------------------
|
| 71 |
DEFAULT_API_BASE_URL = "https://router.huggingface.co/v1"
|
| 72 |
-
DEFAULT_MODEL_NAME = "
|
| 73 |
|
| 74 |
|
| 75 |
def _get_int_env(name: str, default: int) -> int:
|
|
@@ -85,12 +85,12 @@ def _get_int_env(name: str, default: int) -> int:
|
|
| 85 |
)
|
| 86 |
return default
|
| 87 |
|
| 88 |
-
API_BASE_URL = os.getenv("API_BASE_URL"
|
| 89 |
-
MODEL_NAME = os.getenv("MODEL_NAME"
|
| 90 |
HF_TOKEN = os.getenv("HF_TOKEN")
|
| 91 |
-
API_KEY = os.getenv("API_KEY") or HF_TOKEN
|
| 92 |
LOCAL_IMAGE_NAME = os.getenv("LOCAL_IMAGE_NAME")
|
| 93 |
-
ENV_URL = os.getenv("ENV_URL"
|
| 94 |
|
| 95 |
SEED = _get_int_env("SEED", 42)
|
| 96 |
TASK_ID_ENV = os.getenv("TASK_ID")
|
|
@@ -106,7 +106,7 @@ RUN_ALL_TASKS_ENV = os.getenv("RUN_ALL_TASKS", "").strip().lower() in {
|
|
| 106 |
|
| 107 |
|
| 108 |
def llm_mode_enabled() -> bool:
|
| 109 |
-
return bool(API_KEY) and MODEL_NAME
|
| 110 |
|
| 111 |
|
| 112 |
llm_client: OpenAI | None = None
|
|
|
|
| 14 |
|
| 15 |
MODEL_NAME
|
| 16 |
Model identifier to use for LLM inference.
|
| 17 |
+
Default: ``gpt-4o-mini``
|
| 18 |
|
| 19 |
API_KEY
|
| 20 |
Proxy/API authentication token injected by the evaluator.
|
|
|
|
| 69 |
# Configuration
|
| 70 |
# ---------------------------------------------------------------------------
|
| 71 |
DEFAULT_API_BASE_URL = "https://router.huggingface.co/v1"
|
| 72 |
+
DEFAULT_MODEL_NAME = "gpt-4o-mini"
|
| 73 |
|
| 74 |
|
| 75 |
def _get_int_env(name: str, default: int) -> int:
|
|
|
|
| 85 |
)
|
| 86 |
return default
|
| 87 |
|
| 88 |
+
API_BASE_URL = (os.getenv("API_BASE_URL") or DEFAULT_API_BASE_URL).strip()
|
| 89 |
+
MODEL_NAME = (os.getenv("MODEL_NAME") or DEFAULT_MODEL_NAME).strip()
|
| 90 |
HF_TOKEN = os.getenv("HF_TOKEN")
|
| 91 |
+
API_KEY = (os.getenv("API_KEY") or HF_TOKEN or "").strip() or None
|
| 92 |
LOCAL_IMAGE_NAME = os.getenv("LOCAL_IMAGE_NAME")
|
| 93 |
+
ENV_URL = (os.getenv("ENV_URL") or "http://localhost:7860").strip()
|
| 94 |
|
| 95 |
SEED = _get_int_env("SEED", 42)
|
| 96 |
TASK_ID_ENV = os.getenv("TASK_ID")
|
|
|
|
| 106 |
|
| 107 |
|
| 108 |
def llm_mode_enabled() -> bool:
|
| 109 |
+
return bool(API_KEY) and bool(MODEL_NAME)
|
| 110 |
|
| 111 |
|
| 112 |
llm_client: OpenAI | None = None
|
tests/test_inference_unit.py
CHANGED
|
@@ -136,21 +136,17 @@ class InferenceUnitTests(unittest.TestCase):
|
|
| 136 |
inference.API_BASE_URL,
|
| 137 |
"https://router.huggingface.co/v1",
|
| 138 |
)
|
| 139 |
-
self.assertEqual(inference.MODEL_NAME, "
|
| 140 |
self.assertIsNone(inference.API_KEY)
|
| 141 |
self.assertIsNone(inference.HF_TOKEN)
|
| 142 |
self.assertFalse(inference.llm_mode_enabled())
|
| 143 |
|
| 144 |
def test_api_key_enables_llm_mode_without_hf_token(self) -> None:
|
| 145 |
-
inference = _load_inference_module(
|
| 146 |
-
{
|
| 147 |
-
"API_KEY": "validator-proxy-key",
|
| 148 |
-
"MODEL_NAME": "meta/test-model",
|
| 149 |
-
}
|
| 150 |
-
)
|
| 151 |
|
| 152 |
self.assertEqual(inference.API_KEY, "validator-proxy-key")
|
| 153 |
self.assertIsNone(inference.HF_TOKEN)
|
|
|
|
| 154 |
self.assertTrue(inference.llm_mode_enabled())
|
| 155 |
|
| 156 |
def test_seed_env_override_is_respected(self) -> None:
|
|
|
|
| 136 |
inference.API_BASE_URL,
|
| 137 |
"https://router.huggingface.co/v1",
|
| 138 |
)
|
| 139 |
+
self.assertEqual(inference.MODEL_NAME, "gpt-4o-mini")
|
| 140 |
self.assertIsNone(inference.API_KEY)
|
| 141 |
self.assertIsNone(inference.HF_TOKEN)
|
| 142 |
self.assertFalse(inference.llm_mode_enabled())
|
| 143 |
|
| 144 |
def test_api_key_enables_llm_mode_without_hf_token(self) -> None:
|
| 145 |
+
inference = _load_inference_module({"API_KEY": "validator-proxy-key"})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 146 |
|
| 147 |
self.assertEqual(inference.API_KEY, "validator-proxy-key")
|
| 148 |
self.assertIsNone(inference.HF_TOKEN)
|
| 149 |
+
self.assertEqual(inference.MODEL_NAME, "gpt-4o-mini")
|
| 150 |
self.assertTrue(inference.llm_mode_enabled())
|
| 151 |
|
| 152 |
def test_seed_env_override_is_respected(self) -> None:
|