Spaces:
Sleeping
Sleeping
Sync from GitHub 408f81efe5d73e4f9e0623f822105a8849698a0c
Browse files- config.py +7 -6
- tests/test_master_transfer_integrations.py +8 -8
- tests/test_uat_llm_judge.py +26 -1
config.py
CHANGED
|
@@ -33,12 +33,13 @@ MAX_CONVERSATION_TURNS = 20
|
|
| 33 |
# ============================================ LLM Configuration ============================================
|
| 34 |
|
| 35 |
# Each role-specific model configuration is defined as (PROVIDER, MODEL_NAME).
|
| 36 |
-
#
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
|
|
|
| 42 |
|
| 43 |
# Legacy defaults retained for compatibility with existing callers.
|
| 44 |
LLM_PROVIDER = 'openai'
|
|
|
|
| 33 |
# ============================================ LLM Configuration ============================================
|
| 34 |
|
| 35 |
# Each role-specific model configuration is defined as (PROVIDER, MODEL_NAME).
|
| 36 |
+
# All roles run through OpenRouter (OpenAI-compatible) so the app no longer
|
| 37 |
+
# depends on a separately funded OpenAI account.
|
| 38 |
+
MAIN_AGENT_MODEL = ('open_router:openai', 'openai/gpt-4.1')
|
| 39 |
+
FALLBACK_MODELS = [('open_router:openai', 'openai/gpt-4o-mini')]
|
| 40 |
+
LANGUAGE_DETECTION_MODEL = ('open_router:openai', 'openai/gpt-4o-mini')
|
| 41 |
+
CONFIDENCE_SCORING_MODEL = ('open_router:openai', 'openai/gpt-4o-mini')
|
| 42 |
+
SUMMARIZATION_MODEL = ('open_router:openai', 'openai/gpt-4.1')
|
| 43 |
|
| 44 |
# Legacy defaults retained for compatibility with existing callers.
|
| 45 |
LLM_PROVIDER = 'openai'
|
tests/test_master_transfer_integrations.py
CHANGED
|
@@ -234,7 +234,7 @@ def test_query_falls_back_to_bm25_when_embedding_fails(monkeypatch):
|
|
| 234 |
assert collection.query.bm25_calls[0]["limit"] == 3
|
| 235 |
|
| 236 |
|
| 237 |
-
def
|
| 238 |
calls = []
|
| 239 |
|
| 240 |
def fake_initialize_model(cls, provider, model, role="main"):
|
|
@@ -252,10 +252,10 @@ def test_model_config_keeps_master_defaults_and_budgets(monkeypatch):
|
|
| 252 |
ModelConfigurator.get_confidence_scoring_model()
|
| 253 |
ModelConfigurator.get_fallback_models()
|
| 254 |
|
| 255 |
-
|
| 256 |
-
|
| 257 |
-
assert config.llm.
|
| 258 |
-
assert config.llm.
|
| 259 |
assert ModelConfigurator._openai_budget("main") == {
|
| 260 |
"max_tokens": 3072,
|
| 261 |
"timeout": 30,
|
|
@@ -266,6 +266,6 @@ def test_model_config_keeps_master_defaults_and_budgets(monkeypatch):
|
|
| 266 |
"timeout": 10,
|
| 267 |
"request_timeout": 10,
|
| 268 |
}
|
| 269 |
-
assert ("openai", "gpt-4.1", "main") in calls
|
| 270 |
-
assert ("openai", "gpt-4o-mini", "language_detector") in calls
|
| 271 |
-
assert ("openai", "gpt-4o-mini", "confidence_scoring") in calls
|
|
|
|
| 234 |
assert collection.query.bm25_calls[0]["limit"] == 3
|
| 235 |
|
| 236 |
|
| 237 |
+
def test_model_config_uses_openrouter_defaults_and_budgets(monkeypatch):
|
| 238 |
calls = []
|
| 239 |
|
| 240 |
def fake_initialize_model(cls, provider, model, role="main"):
|
|
|
|
| 252 |
ModelConfigurator.get_confidence_scoring_model()
|
| 253 |
ModelConfigurator.get_fallback_models()
|
| 254 |
|
| 255 |
+
# All roles run through OpenRouter so the app no longer needs a funded
|
| 256 |
+
# OpenAI account.
|
| 257 |
+
assert config.llm.MAIN_AGENT_MODEL == ("open_router:openai", "openai/gpt-4.1")
|
| 258 |
+
assert config.llm.FALLBACK_MODELS == [("open_router:openai", "openai/gpt-4o-mini")]
|
| 259 |
assert ModelConfigurator._openai_budget("main") == {
|
| 260 |
"max_tokens": 3072,
|
| 261 |
"timeout": 30,
|
|
|
|
| 266 |
"timeout": 10,
|
| 267 |
"request_timeout": 10,
|
| 268 |
}
|
| 269 |
+
assert ("open_router:openai", "openai/gpt-4.1", "main") in calls
|
| 270 |
+
assert ("open_router:openai", "openai/gpt-4o-mini", "language_detector") in calls
|
| 271 |
+
assert ("open_router:openai", "openai/gpt-4o-mini", "confidence_scoring") in calls
|
tests/test_uat_llm_judge.py
CHANGED
|
@@ -25,9 +25,27 @@ from src.config import config
|
|
| 25 |
from src.rag.agent_chain import ExecutiveAgentChain
|
| 26 |
|
| 27 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 28 |
DEFAULT_EXCEL = Path(__file__).resolve().parent / "fixtures" / "UAT.xlsx"
|
| 29 |
SKIPPED_SHEETS = {"About", "TestProtocol", "Reporting"}
|
| 30 |
-
DEFAULT_JUDGE_MODEL = os.getenv("UAT_JUDGE_MODEL"
|
|
|
|
|
|
|
| 31 |
MIN_ACCEPTABLE_SCORE = float(os.getenv("UAT_MIN_SCORE", "7.0"))
|
| 32 |
|
| 33 |
|
|
@@ -264,6 +282,13 @@ def _openai_client():
|
|
| 264 |
except ImportError as exc:
|
| 265 |
raise RuntimeError("openai is required for the UAT LLM judge.") from exc
|
| 266 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 267 |
api_key = os.getenv("OPENAI_API_KEY") or os.getenv("OPENAI_COMPAT_API_KEY")
|
| 268 |
base_url = os.getenv("OPENAI_COMPAT_BASE_URL")
|
| 269 |
if not api_key:
|
|
|
|
| 25 |
from src.rag.agent_chain import ExecutiveAgentChain
|
| 26 |
|
| 27 |
|
| 28 |
+
OPENROUTER_BASE_URL = "https://openrouter.ai/api/v1"
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def _use_openrouter() -> bool:
|
| 32 |
+
"""Route the LLM judge through OpenRouter by default (opt out with
|
| 33 |
+
UAT_USE_OPENROUTER=0).
|
| 34 |
+
|
| 35 |
+
The chatbot's own models already run through OpenRouter via config.py; this
|
| 36 |
+
keeps the judge consistent so the UAT does not depend on a funded OpenAI
|
| 37 |
+
account.
|
| 38 |
+
"""
|
| 39 |
+
if os.getenv("UAT_USE_OPENROUTER") == "0":
|
| 40 |
+
return False
|
| 41 |
+
return bool(os.getenv("OPEN_ROUTER_API_KEY"))
|
| 42 |
+
|
| 43 |
+
|
| 44 |
DEFAULT_EXCEL = Path(__file__).resolve().parent / "fixtures" / "UAT.xlsx"
|
| 45 |
SKIPPED_SHEETS = {"About", "TestProtocol", "Reporting"}
|
| 46 |
+
DEFAULT_JUDGE_MODEL = os.getenv("UAT_JUDGE_MODEL") or (
|
| 47 |
+
"openai/gpt-4o-mini" if _use_openrouter() else "gpt-4o-mini"
|
| 48 |
+
)
|
| 49 |
MIN_ACCEPTABLE_SCORE = float(os.getenv("UAT_MIN_SCORE", "7.0"))
|
| 50 |
|
| 51 |
|
|
|
|
| 282 |
except ImportError as exc:
|
| 283 |
raise RuntimeError("openai is required for the UAT LLM judge.") from exc
|
| 284 |
|
| 285 |
+
if _use_openrouter():
|
| 286 |
+
return OpenAI(
|
| 287 |
+
api_key=os.environ["OPEN_ROUTER_API_KEY"],
|
| 288 |
+
base_url=OPENROUTER_BASE_URL,
|
| 289 |
+
timeout=float(os.getenv("UAT_JUDGE_TIMEOUT_S", "120")),
|
| 290 |
+
)
|
| 291 |
+
|
| 292 |
api_key = os.getenv("OPENAI_API_KEY") or os.getenv("OPENAI_COMPAT_API_KEY")
|
| 293 |
base_url = os.getenv("OPENAI_COMPAT_BASE_URL")
|
| 294 |
if not api_key:
|