Mindigenous commited on
Commit
609b79a
·
verified ·
1 Parent(s): 863496b

Tighten clean_output role-prefix regex; debug-flag for raw output

Browse files
Files changed (1) hide show
  1. app.py +14 -1
app.py CHANGED
@@ -50,12 +50,25 @@ def parse_output(text: str) -> dict:
50
  _CHAT_TOKEN_PATTERN = re.compile(
51
  r"<\|(?:im_start|im_end|endoftext|fim_prefix|fim_middle|fim_suffix|fim_pad|repo_name|file_sep)\|>"
52
  )
 
 
 
 
53
 
54
 
55
  def clean_output(text: str) -> str:
56
  """Strip Qwen chat-template artifacts and any leading role prefix."""
 
 
57
  text = _CHAT_TOKEN_PATTERN.sub("", text)
58
- text = re.sub(r"^\s*(system|user|assistant)\s*\n", "", text)
 
 
 
 
 
 
 
59
  return text.strip()
60
 
61
 
 
50
  _CHAT_TOKEN_PATTERN = re.compile(
51
  r"<\|(?:im_start|im_end|endoftext|fim_prefix|fim_middle|fim_suffix|fim_pad|repo_name|file_sep)\|>"
52
  )
53
+ # Match a role line ONLY if it stands alone at the very start of the text
54
+ # followed by an explicit newline. The previous '\s*' wildcard could swallow
55
+ # leading content when the model emitted weird sequences in the vision path.
56
+ _ROLE_PREFIX_PATTERN = re.compile(r"^(?:system|user|assistant)\n")
57
 
58
 
59
  def clean_output(text: str) -> str:
60
  """Strip Qwen chat-template artifacts and any leading role prefix."""
61
+ if os.environ.get("MINDI_DEBUG_RAW") == "1":
62
+ print(f"[clean_output] RAW ({len(text)} chars): {text!r}")
63
  text = _CHAT_TOKEN_PATTERN.sub("", text)
64
+ # Apply role-prefix strip up to twice: handles the vision-path case where
65
+ # the model occasionally emits 'assistant\n' followed by stray noise like
66
+ # an extra 'user\n' before the real reply.
67
+ for _ in range(2):
68
+ new_text = _ROLE_PREFIX_PATTERN.sub("", text, count=1)
69
+ if new_text == text:
70
+ break
71
+ text = new_text
72
  return text.strip()
73
 
74