Muhammad Faraz Malik commited on
Commit ·
4d02a2a
1
Parent(s): e497738
Use Groq native json_mode structured output and increase max_tokens to prevent empty reasoning responses
Browse files- src/agent/llm.py +50 -19
- src/agent/prompts.py +3 -0
src/agent/llm.py
CHANGED
|
@@ -12,7 +12,7 @@ from pathlib import Path
|
|
| 12 |
from typing import TypeVar
|
| 13 |
|
| 14 |
from langchain_core.language_models.chat_models import BaseChatModel
|
| 15 |
-
from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage
|
| 16 |
from langchain_groq import ChatGroq
|
| 17 |
from pydantic import BaseModel, Field
|
| 18 |
from tenacity import (
|
|
@@ -120,16 +120,11 @@ def default_groq_api_key() -> str:
|
|
| 120 |
return os.environ.get("GROQ_API_KEY", "").strip()
|
| 121 |
|
| 122 |
|
| 123 |
-
def default_groq_api_key() -> str:
|
| 124 |
-
load_dotenv_into_environ()
|
| 125 |
-
return os.environ.get("GROQ_API_KEY", "").strip()
|
| 126 |
-
|
| 127 |
-
|
| 128 |
def create_groq_llm(
|
| 129 |
model: str | None = None,
|
| 130 |
api_key: str | None = None,
|
| 131 |
temperature: float = 0.0,
|
| 132 |
-
max_tokens: int =
|
| 133 |
) -> ChatGroq:
|
| 134 |
"""Instantiate `ChatGroq` configured for Groq's free tier."""
|
| 135 |
key = (api_key or default_groq_api_key()).strip()
|
|
@@ -168,7 +163,8 @@ def invoke_with_backoff(llm: BaseChatModel, messages: list[BaseMessage]) -> Base
|
|
| 168 |
|
| 169 |
def _extract_json_object(text: str) -> dict:
|
| 170 |
"""Extract the first JSON object from an LLM response (stripping <think> tags and fences)."""
|
| 171 |
-
cleaned = re.sub(r"<think>.*?</think>", "", text, flags=re.DOTALL)
|
|
|
|
| 172 |
fence = re.search(r"```(?:json)?\s*(\{.*?\})\s*```", cleaned, flags=re.DOTALL)
|
| 173 |
if fence:
|
| 174 |
cleaned = fence.group(1)
|
|
@@ -180,6 +176,18 @@ def _extract_json_object(text: str) -> dict:
|
|
| 180 |
return json.loads(cleaned)
|
| 181 |
|
| 182 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 183 |
def invoke_structured(
|
| 184 |
llm: BaseChatModel,
|
| 185 |
schema: type[T],
|
|
@@ -188,14 +196,38 @@ def invoke_structured(
|
|
| 188 |
) -> T:
|
| 189 |
"""Invoke the LLM and parse its output into ``schema`` (Pydantic v2).
|
| 190 |
|
| 191 |
-
Uses
|
| 192 |
-
`
|
|
|
|
| 193 |
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 194 |
schema_json = json.dumps(schema.model_json_schema(), indent=2)
|
| 195 |
full_system = (
|
| 196 |
f"{system_prompt.strip()}\n\n"
|
| 197 |
-
"Respond ONLY with a single valid JSON object
|
| 198 |
-
"(no markdown prose outside the JSON):\n"
|
| 199 |
f"{schema_json}"
|
| 200 |
)
|
| 201 |
messages: list[BaseMessage] = [
|
|
@@ -203,7 +235,7 @@ def invoke_structured(
|
|
| 203 |
HumanMessage(content=user_prompt),
|
| 204 |
]
|
| 205 |
response = invoke_with_backoff(llm, messages)
|
| 206 |
-
content =
|
| 207 |
try:
|
| 208 |
data = _extract_json_object(content)
|
| 209 |
return schema.model_validate(data)
|
|
@@ -211,17 +243,16 @@ def invoke_structured(
|
|
| 211 |
logger.warning("First structured parse failed (%s); asking LLM to repair JSON", first_err)
|
| 212 |
repair_messages: list[BaseMessage] = [
|
| 213 |
*messages,
|
| 214 |
-
|
| 215 |
HumanMessage(
|
| 216 |
content=(
|
| 217 |
-
f"Your response could not be parsed
|
| 218 |
-
"
|
|
|
|
| 219 |
)
|
| 220 |
),
|
| 221 |
]
|
| 222 |
repaired = invoke_with_backoff(llm, repair_messages)
|
| 223 |
-
repaired_text = (
|
| 224 |
-
repaired.content if isinstance(repaired.content, str) else json.dumps(repaired.content)
|
| 225 |
-
)
|
| 226 |
return schema.model_validate(_extract_json_object(repaired_text))
|
| 227 |
|
|
|
|
| 12 |
from typing import TypeVar
|
| 13 |
|
| 14 |
from langchain_core.language_models.chat_models import BaseChatModel
|
| 15 |
+
from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, SystemMessage
|
| 16 |
from langchain_groq import ChatGroq
|
| 17 |
from pydantic import BaseModel, Field
|
| 18 |
from tenacity import (
|
|
|
|
| 120 |
return os.environ.get("GROQ_API_KEY", "").strip()
|
| 121 |
|
| 122 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 123 |
def create_groq_llm(
|
| 124 |
model: str | None = None,
|
| 125 |
api_key: str | None = None,
|
| 126 |
temperature: float = 0.0,
|
| 127 |
+
max_tokens: int = 4096,
|
| 128 |
) -> ChatGroq:
|
| 129 |
"""Instantiate `ChatGroq` configured for Groq's free tier."""
|
| 130 |
key = (api_key or default_groq_api_key()).strip()
|
|
|
|
| 163 |
|
| 164 |
def _extract_json_object(text: str) -> dict:
|
| 165 |
"""Extract the first JSON object from an LLM response (stripping <think> tags and fences)."""
|
| 166 |
+
cleaned = re.sub(r"<think>.*?</think>", "", text, flags=re.DOTALL)
|
| 167 |
+
cleaned = re.sub(r"<think>.*", "", cleaned, flags=re.DOTALL).strip()
|
| 168 |
fence = re.search(r"```(?:json)?\s*(\{.*?\})\s*```", cleaned, flags=re.DOTALL)
|
| 169 |
if fence:
|
| 170 |
cleaned = fence.group(1)
|
|
|
|
| 176 |
return json.loads(cleaned)
|
| 177 |
|
| 178 |
|
| 179 |
+
def _message_text(msg: BaseMessage) -> str:
|
| 180 |
+
content = msg.content if isinstance(msg.content, str) else json.dumps(msg.content)
|
| 181 |
+
if content and content.strip():
|
| 182 |
+
return content
|
| 183 |
+
# Fallback if reasoning model placed output in additional_kwargs
|
| 184 |
+
extra = getattr(msg, "additional_kwargs", {}) or {}
|
| 185 |
+
for k in ("reasoning_content", "content"):
|
| 186 |
+
if isinstance(extra.get(k), str) and extra[k].strip():
|
| 187 |
+
return extra[k]
|
| 188 |
+
return ""
|
| 189 |
+
|
| 190 |
+
|
| 191 |
def invoke_structured(
|
| 192 |
llm: BaseChatModel,
|
| 193 |
schema: type[T],
|
|
|
|
| 196 |
) -> T:
|
| 197 |
"""Invoke the LLM and parse its output into ``schema`` (Pydantic v2).
|
| 198 |
|
| 199 |
+
Uses native `with_structured_output(schema, method="json_mode")` when supported by
|
| 200 |
+
`ChatGroq` so reasoning models never return empty content, with a resilient manual
|
| 201 |
+
JSON-extraction + repair fallback for custom/fake test models.
|
| 202 |
"""
|
| 203 |
+
if isinstance(llm, ChatGroq):
|
| 204 |
+
try:
|
| 205 |
+
structured_llm = llm.with_structured_output(schema, method="json_mode")
|
| 206 |
+
schema_hint = json.dumps(schema.model_json_schema(), indent=2)
|
| 207 |
+
res = structured_llm.invoke(
|
| 208 |
+
[
|
| 209 |
+
SystemMessage(
|
| 210 |
+
content=(
|
| 211 |
+
f"{system_prompt.strip()}\n\n"
|
| 212 |
+
"Respond with a valid JSON object matching this schema:\n"
|
| 213 |
+
f"{schema_hint}"
|
| 214 |
+
)
|
| 215 |
+
),
|
| 216 |
+
HumanMessage(content=user_prompt),
|
| 217 |
+
]
|
| 218 |
+
)
|
| 219 |
+
if isinstance(res, schema):
|
| 220 |
+
return res
|
| 221 |
+
if isinstance(res, dict):
|
| 222 |
+
return schema.model_validate(res)
|
| 223 |
+
except Exception as native_err:
|
| 224 |
+
logger.warning("Native json_mode structured output failed (%s); falling back", native_err)
|
| 225 |
+
|
| 226 |
schema_json = json.dumps(schema.model_json_schema(), indent=2)
|
| 227 |
full_system = (
|
| 228 |
f"{system_prompt.strip()}\n\n"
|
| 229 |
+
"Be concise in internal reasoning. Respond ONLY with a single valid JSON object "
|
| 230 |
+
"matching this JSON schema (no markdown prose outside the JSON):\n"
|
| 231 |
f"{schema_json}"
|
| 232 |
)
|
| 233 |
messages: list[BaseMessage] = [
|
|
|
|
| 235 |
HumanMessage(content=user_prompt),
|
| 236 |
]
|
| 237 |
response = invoke_with_backoff(llm, messages)
|
| 238 |
+
content = _message_text(response)
|
| 239 |
try:
|
| 240 |
data = _extract_json_object(content)
|
| 241 |
return schema.model_validate(data)
|
|
|
|
| 243 |
logger.warning("First structured parse failed (%s); asking LLM to repair JSON", first_err)
|
| 244 |
repair_messages: list[BaseMessage] = [
|
| 245 |
*messages,
|
| 246 |
+
AIMessage(content=content or "{}"),
|
| 247 |
HumanMessage(
|
| 248 |
content=(
|
| 249 |
+
f"Your previous response could not be parsed ({first_err}). "
|
| 250 |
+
"Do NOT include reasoning or markdown fences. Output ONLY the raw JSON object "
|
| 251 |
+
f"with keys {list(schema.model_fields.keys())}."
|
| 252 |
)
|
| 253 |
),
|
| 254 |
]
|
| 255 |
repaired = invoke_with_backoff(llm, repair_messages)
|
| 256 |
+
repaired_text = _message_text(repaired)
|
|
|
|
|
|
|
| 257 |
return schema.model_validate(_extract_json_object(repaired_text))
|
| 258 |
|
src/agent/prompts.py
CHANGED
|
@@ -11,6 +11,9 @@ that FAILS on the current buggy code and will PASS once the bug is fixed.
|
|
| 11 |
Rules:
|
| 12 |
- Import directly from the repository modules shown in the context.
|
| 13 |
- Keep the test deterministic, fast, and focused on the exact bug described.
|
|
|
|
|
|
|
|
|
|
| 14 |
- Choose a filename like `test_repro_bug.py` in the repository root.
|
| 15 |
"""
|
| 16 |
|
|
|
|
| 11 |
Rules:
|
| 12 |
- Import directly from the repository modules shown in the context.
|
| 13 |
- Keep the test deterministic, fast, and focused on the exact bug described.
|
| 14 |
+
- If a module reads environment variables or external clients at import time, set dummy env vars
|
| 15 |
+
(`os.environ.setdefault(...)`) or use `unittest.mock.patch` before importing so the test runs
|
| 16 |
+
offline without external API keys.
|
| 17 |
- Choose a filename like `test_repro_bug.py` in the repository root.
|
| 18 |
"""
|
| 19 |
|