Spaces:
Runtime error
Runtime error
Tighten clean_output role-prefix regex; debug-flag for raw output
Browse files
app.py
CHANGED
|
@@ -50,12 +50,25 @@ def parse_output(text: str) -> dict:
|
|
| 50 |
_CHAT_TOKEN_PATTERN = re.compile(
|
| 51 |
r"<\|(?:im_start|im_end|endoftext|fim_prefix|fim_middle|fim_suffix|fim_pad|repo_name|file_sep)\|>"
|
| 52 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 53 |
|
| 54 |
|
| 55 |
def clean_output(text: str) -> str:
|
| 56 |
"""Strip Qwen chat-template artifacts and any leading role prefix."""
|
|
|
|
|
|
|
| 57 |
text = _CHAT_TOKEN_PATTERN.sub("", text)
|
| 58 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 59 |
return text.strip()
|
| 60 |
|
| 61 |
|
|
|
|
| 50 |
_CHAT_TOKEN_PATTERN = re.compile(
|
| 51 |
r"<\|(?:im_start|im_end|endoftext|fim_prefix|fim_middle|fim_suffix|fim_pad|repo_name|file_sep)\|>"
|
| 52 |
)
|
| 53 |
+
# Match a role line ONLY if it stands alone at the very start of the text
|
| 54 |
+
# followed by an explicit newline. The previous '\s*' wildcard could swallow
|
| 55 |
+
# leading content when the model emitted weird sequences in the vision path.
|
| 56 |
+
_ROLE_PREFIX_PATTERN = re.compile(r"^(?:system|user|assistant)\n")
|
| 57 |
|
| 58 |
|
| 59 |
def clean_output(text: str) -> str:
|
| 60 |
"""Strip Qwen chat-template artifacts and any leading role prefix."""
|
| 61 |
+
if os.environ.get("MINDI_DEBUG_RAW") == "1":
|
| 62 |
+
print(f"[clean_output] RAW ({len(text)} chars): {text!r}")
|
| 63 |
text = _CHAT_TOKEN_PATTERN.sub("", text)
|
| 64 |
+
# Apply role-prefix strip up to twice: handles the vision-path case where
|
| 65 |
+
# the model occasionally emits 'assistant\n' followed by stray noise like
|
| 66 |
+
# an extra 'user\n' before the real reply.
|
| 67 |
+
for _ in range(2):
|
| 68 |
+
new_text = _ROLE_PREFIX_PATTERN.sub("", text, count=1)
|
| 69 |
+
if new_text == text:
|
| 70 |
+
break
|
| 71 |
+
text = new_text
|
| 72 |
return text.strip()
|
| 73 |
|
| 74 |
|