ricklon commited on
Commit
2987995
·
1 Parent(s): 2fa4f01

Strip malformed grounding tags from markdown output

Browse files
Files changed (2) hide show
  1. app.py +19 -1
  2. tests/test_clean_output.py +54 -0
app.py CHANGED
@@ -247,9 +247,27 @@ def clean_output(text, include_images=False):
247
  text = text.replace(match[0], '', 1)
248
  else:
249
  text = re.sub(rf'(?m)^[^\n]*{re.escape(match[0])}[^\n]*\n?', '', text)
250
-
 
251
  return text.strip()
252
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
253
  PREVIEW_CSS = """
254
  <style>
255
  .math-preview {
 
247
  text = text.replace(match[0], '', 1)
248
  else:
249
  text = re.sub(rf'(?m)^[^\n]*{re.escape(match[0])}[^\n]*\n?', '', text)
250
+
251
+ text = _strip_malformed_grounding(text)
252
  return text.strip()
253
 
254
+ def _strip_malformed_grounding(text: str) -> str:
255
+ """Remove incomplete grounding tags that can leak into OCR markdown/text."""
256
+ if not text:
257
+ return ""
258
+
259
+ line_patterns = [
260
+ r'(?m)^[^\n]*<\|ref\|>.*?<\|/ref\|><\|det\|>.*?(?:<\|/det\|>)?[^\n]*\n?',
261
+ r'(?m)^[^\n]*<\|det\|>.*?(?:<\|/det\|>)?[^\n]*\n?',
262
+ r'(?m)^[^\n]*<\|/?ref\|>[^\n]*\n?',
263
+ ]
264
+ for p in line_patterns:
265
+ text = re.sub(p, '', text)
266
+
267
+ text = re.sub(r'<\|/?ref\|>', '', text)
268
+ text = re.sub(r'<\|/?det\|>', '', text)
269
+ return text
270
+
271
  PREVIEW_CSS = """
272
  <style>
273
  .math-preview {
tests/test_clean_output.py ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ast
2
+ import pathlib
3
+ import re
4
+ import unittest
5
+
6
+
7
+ def _load_clean_output():
8
+ app_path = pathlib.Path(__file__).resolve().parents[1] / "app.py"
9
+ source = app_path.read_text(encoding="utf-8")
10
+ module = ast.parse(source, filename=str(app_path))
11
+
12
+ wanted = {
13
+ "_strip_malformed_grounding",
14
+ "clean_output",
15
+ }
16
+ fn_nodes = [n for n in module.body if isinstance(n, ast.FunctionDef) and n.name in wanted]
17
+ fn_nodes.sort(key=lambda n: n.lineno)
18
+
19
+ test_mod = ast.Module(body=fn_nodes, type_ignores=[])
20
+ code = compile(test_mod, filename=str(app_path), mode="exec")
21
+
22
+ scope = {
23
+ "re": re,
24
+ }
25
+ exec(code, scope)
26
+ return scope["clean_output"]
27
+
28
+
29
+ class CleanOutputTests(unittest.TestCase):
30
+ def test_removes_truncated_grounding_artifact_line(self):
31
+ clean_output = _load_clean_output()
32
+ raw = (
33
+ "\\[ \\frac{18x-34}{(2x-3)^2} \\]\n"
34
+ "<|ref|>equation<|/ref|><|det|>[[50, 0, 450, 100]]\n"
35
+ )
36
+
37
+ cleaned = clean_output(raw, include_images=True)
38
+ self.assertIn("\\[ \\frac{18x-34}{(2x-3)^2} \\]", cleaned)
39
+ self.assertNotIn("<|ref|>", cleaned)
40
+ self.assertNotIn("<|det|>", cleaned)
41
+ self.assertNotIn("[[50, 0, 450, 100]]", cleaned)
42
+
43
+ def test_replaces_full_image_reference(self):
44
+ clean_output = _load_clean_output()
45
+ raw = "prefix\n<|ref|>image<|/ref|><|det|>[[0,0,100,100]]<|/det|>\nsuffix"
46
+
47
+ cleaned = clean_output(raw, include_images=True)
48
+ self.assertIn("**[Figure 1]**", cleaned)
49
+ self.assertNotIn("<|ref|>", cleaned)
50
+ self.assertNotIn("<|det|>", cleaned)
51
+
52
+
53
+ if __name__ == "__main__":
54
+ unittest.main()