| """bankml 0.3.6 on top of PR #1: sampling options, the chat/judge models a bankml node names, |
| and the serve flags that layer a persona and its parameters.""" |
|
|
| from __future__ import annotations |
|
|
| import asyncio |
| import json |
|
|
| import httpx |
| import pytest |
|
|
| from mindxtrain.models.registry import ChatMessage, ChatRequest |
| from mindxtrain.operator.backends.bankml import BankmlBackend |
|
|
| _ENV = ("MINDXTRAIN_CHAT_MODEL", "MINDXTRAIN_JUDGE_MODEL", "MINDXTRAIN_CHAT_OPTIONS", |
| "MINDXTRAIN_BANKML_OPTIONS") |
|
|
|
|
| @pytest.fixture(autouse=True) |
| def _clean_env(monkeypatch: pytest.MonkeyPatch) -> None: |
| for k in _ENV: |
| monkeypatch.delenv(k, raising=False) |
|
|
|
|
| def _backend(seen: list[dict[str, object]], **kw: object) -> BankmlBackend: |
| def handler(request: httpx.Request) -> httpx.Response: |
| seen.append(json.loads(request.content)) |
| return httpx.Response(200, json={ |
| "model": "mindx-gen39", "choices": [{"message": {"content": "I am mindX."}, "finish_reason": "stop"}], |
| "usage": {"prompt_tokens": 9, "completion_tokens": 4}, |
| "bankml_receipt": {"model_sha256": "6b64c748"}, |
| }) |
|
|
| return BankmlBackend(transport=httpx.MockTransport(handler), **kw) |
|
|
|
|
| def _ask(b: BankmlBackend) -> None: |
| asyncio.run(b.chat(ChatRequest(model="mindx-gen39", messages=[ChatMessage(role="user", content="hi")], |
| max_tokens=16))) |
|
|
|
|
| def test_backend_sends_options_only_when_set(monkeypatch: pytest.MonkeyPatch) -> None: |
| seen: list[dict[str, object]] = [] |
| _ask(_backend(seen)) |
| assert not {"repeat_penalty", "repeat_last_n"} & set(seen[0]) |
| monkeypatch.setenv("MINDXTRAIN_BANKML_OPTIONS", '{"repeat_penalty": 1.3, "repeat_last_n": 64}') |
| _ask(_backend(seen, options={"repeat_last_n": 32})) |
| assert seen[1]["repeat_penalty"] == 1.3 and seen[1]["repeat_last_n"] == 32 |
| assert seen[1]["model"] == "mindx-gen39" and seen[1]["max_tokens"] == 16 |
|
|
|
|
| def test_backend_options_must_be_an_object(monkeypatch: pytest.MonkeyPatch) -> None: |
| monkeypatch.setenv("MINDXTRAIN_BANKML_OPTIONS", "[1.3]") |
| with pytest.raises(ValueError, match="JSON object"): |
| BankmlBackend() |
|
|
|
|
| def test_chat_and_judge_models_follow_env(monkeypatch: pytest.MonkeyPatch) -> None: |
| from mindxtrain.eval.llama_evals import ( |
| CorrectnessEvaluator, |
| PairwiseEvaluator, |
| default_judge_model, |
| ) |
| from mindxtrain.governance.panel import default_chat_model |
|
|
| assert default_chat_model() == "llama3.2" and default_judge_model() == "llama3.2" |
| monkeypatch.setenv("MINDXTRAIN_CHAT_MODEL", "bonsai-8b-q1_0") |
| assert default_judge_model() == "bonsai-8b-q1_0" |
| monkeypatch.setenv("MINDXTRAIN_JUDGE_MODEL", "bonsai-1.7b") |
| assert CorrectnessEvaluator().model == "bonsai-1.7b" |
| assert PairwiseEvaluator(model="explicit").model == "explicit" |
|
|
|
|
| def test_chat_once_options(monkeypatch: pytest.MonkeyPatch) -> None: |
| from mindxtrain.governance import panel |
|
|
| bodies: list[dict[str, object]] = [] |
|
|
| def handler(request: httpx.Request) -> httpx.Response: |
| bodies.append(json.loads(request.content)) |
| return httpx.Response(200, json={"choices": [{"message": {"content": "VERDICT: APPROVE"}}]}) |
|
|
| real = httpx.Client |
| monkeypatch.setattr(httpx, "Client", lambda **k: real(transport=httpx.MockTransport(handler), **k)) |
| url = "http://127.0.0.1:18093/v1" |
| panel.chat_once("m", [{"role": "user", "content": "x"}], base_url=url) |
| monkeypatch.setenv("MINDXTRAIN_CHAT_OPTIONS", '{"repeat_penalty": 1.3}') |
| panel.chat_once("m", [{"role": "user", "content": "x"}], base_url=url) |
| panel.chat_once("m", [{"role": "user", "content": "x"}], base_url=url, options={}) |
| assert "repeat_penalty" not in bodies[0] and bodies[1]["repeat_penalty"] == 1.3 |
| assert "repeat_penalty" not in bodies[2] |
| monkeypatch.setenv("MINDXTRAIN_CHAT_OPTIONS", "nope") |
| with pytest.raises(ValueError, match="not JSON"): |
| panel.chat_options() |
|
|
|
|
| def test_deliberate_uses_the_env_chat_model(monkeypatch: pytest.MonkeyPatch) -> None: |
| from mindxtrain.governance import panel |
| from mindxtrain.governance.boardroom import Member |
|
|
| asked: list[str] = [] |
| monkeypatch.setattr(panel, "chat_once", lambda model, *a, **k: asked.append(model) or "VERDICT: APPROVE") |
| monkeypatch.setenv("MINDXTRAIN_CHAT_MODEL", "bonsai-8b-q1_0") |
| d = panel.deliberate(Member(id="m1", role="critic"), "ship it") |
| assert asked == ["bonsai-8b-q1_0"] and d.vote == "approve" |
|
|
|
|
| def test_serve_param_parsing() -> None: |
| from mindxtrain.cli.main import _parse_params |
|
|
| assert _parse_params(["repeat_penalty=1.3", "num_ctx=2048", "seed=0"]) == { |
| "repeat_penalty": 1.3, "num_ctx": 2048, "seed": 0} |
| with pytest.raises(ValueError): |
| _parse_params(["repeat_penalty"]) |
|
|
