Improve GPT-OSS writing speed and reliability
Browse files- services/cowriter_service.py +4 -2
- services/llm_client.py +5 -0
- tests/test_llm_client.py +23 -1
services/cowriter_service.py
CHANGED
|
@@ -72,7 +72,9 @@ def _suggest_llm(
|
|
| 72 |
system_prompt=system_prompt,
|
| 73 |
user_prompt=user_prompt,
|
| 74 |
temperature=0.78,
|
| 75 |
-
|
|
|
|
|
|
|
| 76 |
)
|
| 77 |
model_used = "standard"
|
| 78 |
else:
|
|
@@ -81,7 +83,7 @@ def _suggest_llm(
|
|
| 81 |
user_prompt=user_prompt,
|
| 82 |
model=model,
|
| 83 |
temperature=0.78,
|
| 84 |
-
max_tokens=max_tokens * n +
|
| 85 |
)
|
| 86 |
|
| 87 |
return _parse_suggestions(raw, text, n), model_used
|
|
|
|
| 72 |
system_prompt=system_prompt,
|
| 73 |
user_prompt=user_prompt,
|
| 74 |
temperature=0.78,
|
| 75 |
+
# GPT-OSS counts reasoning and visible text in this budget. Keep a
|
| 76 |
+
# floor so all requested insert-ready options reach the response.
|
| 77 |
+
max_tokens=max(1024, max_tokens * n + 220),
|
| 78 |
)
|
| 79 |
model_used = "standard"
|
| 80 |
else:
|
|
|
|
| 83 |
user_prompt=user_prompt,
|
| 84 |
model=model,
|
| 85 |
temperature=0.78,
|
| 86 |
+
max_tokens=max(1024, max_tokens * n + 220),
|
| 87 |
)
|
| 88 |
|
| 89 |
return _parse_suggestions(raw, text, n), model_used
|
services/llm_client.py
CHANGED
|
@@ -227,6 +227,11 @@ def _call_provider(
|
|
| 227 |
"temperature": temperature,
|
| 228 |
"max_tokens": max_tokens,
|
| 229 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 230 |
|
| 231 |
try:
|
| 232 |
resp = httpx.post(url, json=payload, headers=headers, timeout=timeout)
|
|
|
|
| 227 |
"temperature": temperature,
|
| 228 |
"max_tokens": max_tokens,
|
| 229 |
}
|
| 230 |
+
if model.startswith("openai/gpt-oss-"):
|
| 231 |
+
# These models default to medium reasoning, which spends extra tokens
|
| 232 |
+
# and latency on routine writing tasks. Low effort keeps their quality
|
| 233 |
+
# while returning visible copy much faster and more reliably.
|
| 234 |
+
payload["reasoning_effort"] = "low"
|
| 235 |
|
| 236 |
try:
|
| 237 |
resp = httpx.post(url, json=payload, headers=headers, timeout=timeout)
|
tests/test_llm_client.py
CHANGED
|
@@ -1,8 +1,10 @@
|
|
| 1 |
"""Tests for the current writing-model registry and legacy migrations."""
|
| 2 |
|
|
|
|
|
|
|
| 3 |
import pytest
|
| 4 |
|
| 5 |
-
from services.llm_client import resolve_premium_model
|
| 6 |
|
| 7 |
|
| 8 |
@pytest.mark.parametrize(
|
|
@@ -25,3 +27,23 @@ def test_resolve_premium_model(selector, expected):
|
|
| 25 |
def test_resolve_premium_model_rejects_arbitrary_provider_ids():
|
| 26 |
with pytest.raises(ValueError, match="Unsupported writing model"):
|
| 27 |
resolve_premium_model("unknown/provider-model")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
"""Tests for the current writing-model registry and legacy migrations."""
|
| 2 |
|
| 3 |
+
from unittest.mock import Mock, patch
|
| 4 |
+
|
| 5 |
import pytest
|
| 6 |
|
| 7 |
+
from services.llm_client import _call_provider, resolve_premium_model
|
| 8 |
|
| 9 |
|
| 10 |
@pytest.mark.parametrize(
|
|
|
|
| 27 |
def test_resolve_premium_model_rejects_arbitrary_provider_ids():
|
| 28 |
with pytest.raises(ValueError, match="Unsupported writing model"):
|
| 29 |
resolve_premium_model("unknown/provider-model")
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def test_gpt_oss_requests_low_reasoning_for_fast_writing():
|
| 33 |
+
response = Mock()
|
| 34 |
+
response.raise_for_status.return_value = None
|
| 35 |
+
response.json.return_value = {"choices": [{"message": {"content": "Ready"}}]}
|
| 36 |
+
|
| 37 |
+
with patch("services.llm_client.httpx.post", return_value=response) as post:
|
| 38 |
+
result = _call_provider(
|
| 39 |
+
url="https://provider.test/chat",
|
| 40 |
+
api_key="secret",
|
| 41 |
+
model="openai/gpt-oss-20b",
|
| 42 |
+
messages=[{"role": "user", "content": "Continue this draft"}],
|
| 43 |
+
temperature=0.7,
|
| 44 |
+
max_tokens=1024,
|
| 45 |
+
timeout=30.0,
|
| 46 |
+
)
|
| 47 |
+
|
| 48 |
+
assert result == "Ready"
|
| 49 |
+
assert post.call_args.kwargs["json"]["reasoning_effort"] == "low"
|