Muhammad Faraz Malik commited on
Commit
4d02a2a
·
1 Parent(s): e497738

Use Groq native json_mode structured output and increase max_tokens to prevent empty reasoning responses

Browse files
Files changed (2) hide show
  1. src/agent/llm.py +50 -19
  2. src/agent/prompts.py +3 -0
src/agent/llm.py CHANGED
@@ -12,7 +12,7 @@ from pathlib import Path
12
  from typing import TypeVar
13
 
14
  from langchain_core.language_models.chat_models import BaseChatModel
15
- from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage
16
  from langchain_groq import ChatGroq
17
  from pydantic import BaseModel, Field
18
  from tenacity import (
@@ -120,16 +120,11 @@ def default_groq_api_key() -> str:
120
  return os.environ.get("GROQ_API_KEY", "").strip()
121
 
122
 
123
- def default_groq_api_key() -> str:
124
- load_dotenv_into_environ()
125
- return os.environ.get("GROQ_API_KEY", "").strip()
126
-
127
-
128
  def create_groq_llm(
129
  model: str | None = None,
130
  api_key: str | None = None,
131
  temperature: float = 0.0,
132
- max_tokens: int = 1800,
133
  ) -> ChatGroq:
134
  """Instantiate `ChatGroq` configured for Groq's free tier."""
135
  key = (api_key or default_groq_api_key()).strip()
@@ -168,7 +163,8 @@ def invoke_with_backoff(llm: BaseChatModel, messages: list[BaseMessage]) -> Base
168
 
169
  def _extract_json_object(text: str) -> dict:
170
  """Extract the first JSON object from an LLM response (stripping <think> tags and fences)."""
171
- cleaned = re.sub(r"<think>.*?</think>", "", text, flags=re.DOTALL).strip()
 
172
  fence = re.search(r"```(?:json)?\s*(\{.*?\})\s*```", cleaned, flags=re.DOTALL)
173
  if fence:
174
  cleaned = fence.group(1)
@@ -180,6 +176,18 @@ def _extract_json_object(text: str) -> dict:
180
  return json.loads(cleaned)
181
 
182
 
 
 
 
 
 
 
 
 
 
 
 
 
183
  def invoke_structured(
184
  llm: BaseChatModel,
185
  schema: type[T],
@@ -188,14 +196,38 @@ def invoke_structured(
188
  ) -> T:
189
  """Invoke the LLM and parse its output into ``schema`` (Pydantic v2).
190
 
191
- Uses JSON-schema prompting + resilient JSON extraction so it works reliably across
192
- `openai/gpt-oss-120b`, `qwen/qwen3.8-27b`, and fake test models alike.
 
193
  """
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
194
  schema_json = json.dumps(schema.model_json_schema(), indent=2)
195
  full_system = (
196
  f"{system_prompt.strip()}\n\n"
197
- "Respond ONLY with a single valid JSON object matching this JSON schema "
198
- "(no markdown prose outside the JSON):\n"
199
  f"{schema_json}"
200
  )
201
  messages: list[BaseMessage] = [
@@ -203,7 +235,7 @@ def invoke_structured(
203
  HumanMessage(content=user_prompt),
204
  ]
205
  response = invoke_with_backoff(llm, messages)
206
- content = response.content if isinstance(response.content, str) else json.dumps(response.content)
207
  try:
208
  data = _extract_json_object(content)
209
  return schema.model_validate(data)
@@ -211,17 +243,16 @@ def invoke_structured(
211
  logger.warning("First structured parse failed (%s); asking LLM to repair JSON", first_err)
212
  repair_messages: list[BaseMessage] = [
213
  *messages,
214
- response,
215
  HumanMessage(
216
  content=(
217
- f"Your response could not be parsed against the schema ({first_err}). "
218
- "Return ONLY the corrected raw JSON object."
 
219
  )
220
  ),
221
  ]
222
  repaired = invoke_with_backoff(llm, repair_messages)
223
- repaired_text = (
224
- repaired.content if isinstance(repaired.content, str) else json.dumps(repaired.content)
225
- )
226
  return schema.model_validate(_extract_json_object(repaired_text))
227
 
 
12
  from typing import TypeVar
13
 
14
  from langchain_core.language_models.chat_models import BaseChatModel
15
+ from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, SystemMessage
16
  from langchain_groq import ChatGroq
17
  from pydantic import BaseModel, Field
18
  from tenacity import (
 
120
  return os.environ.get("GROQ_API_KEY", "").strip()
121
 
122
 
 
 
 
 
 
123
  def create_groq_llm(
124
  model: str | None = None,
125
  api_key: str | None = None,
126
  temperature: float = 0.0,
127
+ max_tokens: int = 4096,
128
  ) -> ChatGroq:
129
  """Instantiate `ChatGroq` configured for Groq's free tier."""
130
  key = (api_key or default_groq_api_key()).strip()
 
163
 
164
  def _extract_json_object(text: str) -> dict:
165
  """Extract the first JSON object from an LLM response (stripping <think> tags and fences)."""
166
+ cleaned = re.sub(r"<think>.*?</think>", "", text, flags=re.DOTALL)
167
+ cleaned = re.sub(r"<think>.*", "", cleaned, flags=re.DOTALL).strip()
168
  fence = re.search(r"```(?:json)?\s*(\{.*?\})\s*```", cleaned, flags=re.DOTALL)
169
  if fence:
170
  cleaned = fence.group(1)
 
176
  return json.loads(cleaned)
177
 
178
 
179
+ def _message_text(msg: BaseMessage) -> str:
180
+ content = msg.content if isinstance(msg.content, str) else json.dumps(msg.content)
181
+ if content and content.strip():
182
+ return content
183
+ # Fallback if reasoning model placed output in additional_kwargs
184
+ extra = getattr(msg, "additional_kwargs", {}) or {}
185
+ for k in ("reasoning_content", "content"):
186
+ if isinstance(extra.get(k), str) and extra[k].strip():
187
+ return extra[k]
188
+ return ""
189
+
190
+
191
  def invoke_structured(
192
  llm: BaseChatModel,
193
  schema: type[T],
 
196
  ) -> T:
197
  """Invoke the LLM and parse its output into ``schema`` (Pydantic v2).
198
 
199
+ Uses native `with_structured_output(schema, method="json_mode")` when supported by
200
+ `ChatGroq` so reasoning models never return empty content, with a resilient manual
201
+ JSON-extraction + repair fallback for custom/fake test models.
202
  """
203
+ if isinstance(llm, ChatGroq):
204
+ try:
205
+ structured_llm = llm.with_structured_output(schema, method="json_mode")
206
+ schema_hint = json.dumps(schema.model_json_schema(), indent=2)
207
+ res = structured_llm.invoke(
208
+ [
209
+ SystemMessage(
210
+ content=(
211
+ f"{system_prompt.strip()}\n\n"
212
+ "Respond with a valid JSON object matching this schema:\n"
213
+ f"{schema_hint}"
214
+ )
215
+ ),
216
+ HumanMessage(content=user_prompt),
217
+ ]
218
+ )
219
+ if isinstance(res, schema):
220
+ return res
221
+ if isinstance(res, dict):
222
+ return schema.model_validate(res)
223
+ except Exception as native_err:
224
+ logger.warning("Native json_mode structured output failed (%s); falling back", native_err)
225
+
226
  schema_json = json.dumps(schema.model_json_schema(), indent=2)
227
  full_system = (
228
  f"{system_prompt.strip()}\n\n"
229
+ "Be concise in internal reasoning. Respond ONLY with a single valid JSON object "
230
+ "matching this JSON schema (no markdown prose outside the JSON):\n"
231
  f"{schema_json}"
232
  )
233
  messages: list[BaseMessage] = [
 
235
  HumanMessage(content=user_prompt),
236
  ]
237
  response = invoke_with_backoff(llm, messages)
238
+ content = _message_text(response)
239
  try:
240
  data = _extract_json_object(content)
241
  return schema.model_validate(data)
 
243
  logger.warning("First structured parse failed (%s); asking LLM to repair JSON", first_err)
244
  repair_messages: list[BaseMessage] = [
245
  *messages,
246
+ AIMessage(content=content or "{}"),
247
  HumanMessage(
248
  content=(
249
+ f"Your previous response could not be parsed ({first_err}). "
250
+ "Do NOT include reasoning or markdown fences. Output ONLY the raw JSON object "
251
+ f"with keys {list(schema.model_fields.keys())}."
252
  )
253
  ),
254
  ]
255
  repaired = invoke_with_backoff(llm, repair_messages)
256
+ repaired_text = _message_text(repaired)
 
 
257
  return schema.model_validate(_extract_json_object(repaired_text))
258
 
src/agent/prompts.py CHANGED
@@ -11,6 +11,9 @@ that FAILS on the current buggy code and will PASS once the bug is fixed.
11
  Rules:
12
  - Import directly from the repository modules shown in the context.
13
  - Keep the test deterministic, fast, and focused on the exact bug described.
 
 
 
14
  - Choose a filename like `test_repro_bug.py` in the repository root.
15
  """
16
 
 
11
  Rules:
12
  - Import directly from the repository modules shown in the context.
13
  - Keep the test deterministic, fast, and focused on the exact bug described.
14
+ - If a module reads environment variables or external clients at import time, set dummy env vars
15
+ (`os.environ.setdefault(...)`) or use `unittest.mock.patch` before importing so the test runs
16
+ offline without external API keys.
17
  - Choose a filename like `test_repro_bug.py` in the repository root.
18
  """
19