Spaces:
Sleeping
Sleeping
File size: 5,108 Bytes
1bcb9d8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 | """Per-task backend profile resolution and provider dispatch."""
import importlib
import pytest
@pytest.fixture()
def config(monkeypatch):
# Clear any default LLM_* so the default backend is deterministic.
for k in ("LLM_PROVIDER", "LLM_BASE_URL", "LLM_MODEL", "LLM_API_KEY", "ANTHROPIC_API_KEY"):
monkeypatch.delenv(k, raising=False)
import app.config as config
return importlib.reload(config)
def test_default_backend(config):
be = config.backend(None)
assert be.provider == "openai"
assert be.base_url == "https://api.anthropic.com/v1/"
assert be.model == "claude-haiku-4-5"
assert config.backend("") is config.DEFAULT_BACKEND
def test_named_backend_resolves(config, monkeypatch):
monkeypatch.setenv("LLM_OUMI_PROVIDER", "anthropic")
monkeypatch.setenv("LLM_OUMI_BASE_URL", "https://api.oumi.ai/inference")
monkeypatch.setenv("LLM_OUMI_MODEL", "projects/X/deployments/22")
monkeypatch.setenv("LLM_OUMI_API_KEY", "oumi_test")
be = config.backend("oumi")
assert be.provider == "anthropic"
assert be.base_url == "https://api.oumi.ai/inference"
assert be.model == "projects/X/deployments/22"
assert be.api_key == "oumi_test"
assert be.name == "oumi"
def test_named_provider_defaults_to_openai(config, monkeypatch):
monkeypatch.setenv("LLM_QWEN_BASE_URL", "https://api.oumi.ai/inference/v1/")
monkeypatch.setenv("LLM_QWEN_MODEL", "projects/X/deployments/99")
monkeypatch.setenv("LLM_QWEN_API_KEY", "oumi_test")
assert config.backend("qwen").provider == "openai"
def test_unconfigured_named_backend_falls_back_to_default(config, monkeypatch):
monkeypatch.setenv("LLM_OUMI_BASE_URL", "https://api.oumi.ai/inference")
# MODEL and API_KEY missing -> not fully configured -> default
assert config.backend("oumi") is config.DEFAULT_BACKEND
# ...and the trace says "default", not "oumi", so the fallback isn't silent.
assert config.backend("oumi").name == "default"
def test_task_backend_defaults_none():
from app.tasks.base import Task
t = Task(
id="x", title="X", tagline="", system_prompt="", ui={},
sample=lambda: None, lookup_truth=lambda i: None,
parse_output=lambda s: s, score=lambda tr, p: {}, present=lambda p, tr: {},
)
assert t.backend is None
def test_runner_dispatch_by_provider():
from app import engine
from app.config import Backend
oa = Backend("openai", "u", "m", "k")
an = Backend("anthropic", "u", "m", "k")
assert engine._runner_for(oa) is engine._run_openai
assert engine._runner_for(an) is engine._run_anthropic
class _Block:
"""Stands in for an anthropic content block."""
def __init__(self, **kw):
self.__dict__.update(kw)
def model_dump(self):
return dict(self.__dict__)
def _resp(stop_reason, content):
return _Block(
stop_reason=stop_reason,
content=content,
usage=_Block(input_tokens=1, output_tokens=2),
)
def test_anthropic_run_emits_openai_format_trajectory(monkeypatch):
"""The anthropic path must return the same standard chat record as the
openai path — it's the training-data artifact the UI downloads."""
monkeypatch.setenv("LANGFUSE_TRACING_ENABLED", "false")
monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-lf-test")
monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-lf-test")
from app import engine
from app.config import Backend
from app.tasks.base import Task
task = Task(
id="t", title="T", tagline="", system_prompt="SYS", ui={},
sample=lambda: None, lookup_truth=lambda i: None,
parse_output=lambda s: s, score=lambda tr, p: {}, present=lambda p, tr: {},
tools=[{
"type": "function",
"function": {
"name": "lookup",
"description": "d",
"parameters": {"type": "object", "properties": {"q": {"type": "string"}}},
},
}],
execute_tool=lambda name, args: "TOOL_RESULT",
)
turns = [
_resp("tool_use", [_Block(type="tool_use", id="tu_1", name="lookup", input={"q": "x"})]),
_resp("end_turn", [_Block(type="text", text="DONE")]),
]
client = _Block(messages=_Block(create=lambda **kw: turns.pop(0)))
monkeypatch.setattr(engine, "_anthropic", lambda be: client)
final, transcript, messages = engine._run_anthropic(
task, "hello", Backend("anthropic", "u", "m", "k")
)
assert final == "DONE"
assert transcript == [{"tool": "lookup", "args": {"q": "x"}, "result": "TOOL_RESULT"}]
assert [m["role"] for m in messages] == ["system", "user", "assistant", "tool", "assistant"]
assert messages[0]["content"] == "SYS" and messages[1]["content"] == "hello"
assert messages[2]["tool_calls"] == [{
"id": "tu_1",
"type": "function",
"function": {"name": "lookup", "arguments": '{"q": "x"}'},
}]
assert messages[3] == {"role": "tool", "tool_call_id": "tu_1", "content": "TOOL_RESULT"}
assert messages[-1] == {"role": "assistant", "content": "DONE"}
|