rushabh13 commited on
Commit
3465c03
·
1 Parent(s): 6ab9047

Improve GPT-OSS writing speed and reliability

Browse files
services/cowriter_service.py CHANGED
@@ -72,7 +72,9 @@ def _suggest_llm(
72
  system_prompt=system_prompt,
73
  user_prompt=user_prompt,
74
  temperature=0.78,
75
- max_tokens=max_tokens * n + 180,
 
 
76
  )
77
  model_used = "standard"
78
  else:
@@ -81,7 +83,7 @@ def _suggest_llm(
81
  user_prompt=user_prompt,
82
  model=model,
83
  temperature=0.78,
84
- max_tokens=max_tokens * n + 180,
85
  )
86
 
87
  return _parse_suggestions(raw, text, n), model_used
 
72
  system_prompt=system_prompt,
73
  user_prompt=user_prompt,
74
  temperature=0.78,
75
+ # GPT-OSS counts reasoning and visible text in this budget. Keep a
76
+ # floor so all requested insert-ready options reach the response.
77
+ max_tokens=max(1024, max_tokens * n + 220),
78
  )
79
  model_used = "standard"
80
  else:
 
83
  user_prompt=user_prompt,
84
  model=model,
85
  temperature=0.78,
86
+ max_tokens=max(1024, max_tokens * n + 220),
87
  )
88
 
89
  return _parse_suggestions(raw, text, n), model_used
services/llm_client.py CHANGED
@@ -227,6 +227,11 @@ def _call_provider(
227
  "temperature": temperature,
228
  "max_tokens": max_tokens,
229
  }
 
 
 
 
 
230
 
231
  try:
232
  resp = httpx.post(url, json=payload, headers=headers, timeout=timeout)
 
227
  "temperature": temperature,
228
  "max_tokens": max_tokens,
229
  }
230
+ if model.startswith("openai/gpt-oss-"):
231
+ # These models default to medium reasoning, which spends extra tokens
232
+ # and latency on routine writing tasks. Low effort keeps their quality
233
+ # while returning visible copy much faster and more reliably.
234
+ payload["reasoning_effort"] = "low"
235
 
236
  try:
237
  resp = httpx.post(url, json=payload, headers=headers, timeout=timeout)
tests/test_llm_client.py CHANGED
@@ -1,8 +1,10 @@
1
  """Tests for the current writing-model registry and legacy migrations."""
2
 
 
 
3
  import pytest
4
 
5
- from services.llm_client import resolve_premium_model
6
 
7
 
8
  @pytest.mark.parametrize(
@@ -25,3 +27,23 @@ def test_resolve_premium_model(selector, expected):
25
  def test_resolve_premium_model_rejects_arbitrary_provider_ids():
26
  with pytest.raises(ValueError, match="Unsupported writing model"):
27
  resolve_premium_model("unknown/provider-model")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  """Tests for the current writing-model registry and legacy migrations."""
2
 
3
+ from unittest.mock import Mock, patch
4
+
5
  import pytest
6
 
7
+ from services.llm_client import _call_provider, resolve_premium_model
8
 
9
 
10
  @pytest.mark.parametrize(
 
27
  def test_resolve_premium_model_rejects_arbitrary_provider_ids():
28
  with pytest.raises(ValueError, match="Unsupported writing model"):
29
  resolve_premium_model("unknown/provider-model")
30
+
31
+
32
+ def test_gpt_oss_requests_low_reasoning_for_fast_writing():
33
+ response = Mock()
34
+ response.raise_for_status.return_value = None
35
+ response.json.return_value = {"choices": [{"message": {"content": "Ready"}}]}
36
+
37
+ with patch("services.llm_client.httpx.post", return_value=response) as post:
38
+ result = _call_provider(
39
+ url="https://provider.test/chat",
40
+ api_key="secret",
41
+ model="openai/gpt-oss-20b",
42
+ messages=[{"role": "user", "content": "Continue this draft"}],
43
+ temperature=0.7,
44
+ max_tokens=1024,
45
+ timeout=30.0,
46
+ )
47
+
48
+ assert result == "Ready"
49
+ assert post.call_args.kwargs["json"]["reasoning_effort"] == "low"