Spaces:
Configuration error
Configuration error
Strip malformed grounding tags from markdown output
Browse files- app.py +19 -1
- tests/test_clean_output.py +54 -0
app.py
CHANGED
|
@@ -247,9 +247,27 @@ def clean_output(text, include_images=False):
|
|
| 247 |
text = text.replace(match[0], '', 1)
|
| 248 |
else:
|
| 249 |
text = re.sub(rf'(?m)^[^\n]*{re.escape(match[0])}[^\n]*\n?', '', text)
|
| 250 |
-
|
|
|
|
| 251 |
return text.strip()
|
| 252 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 253 |
PREVIEW_CSS = """
|
| 254 |
<style>
|
| 255 |
.math-preview {
|
|
|
|
| 247 |
text = text.replace(match[0], '', 1)
|
| 248 |
else:
|
| 249 |
text = re.sub(rf'(?m)^[^\n]*{re.escape(match[0])}[^\n]*\n?', '', text)
|
| 250 |
+
|
| 251 |
+
text = _strip_malformed_grounding(text)
|
| 252 |
return text.strip()
|
| 253 |
|
| 254 |
+
def _strip_malformed_grounding(text: str) -> str:
|
| 255 |
+
"""Remove incomplete grounding tags that can leak into OCR markdown/text."""
|
| 256 |
+
if not text:
|
| 257 |
+
return ""
|
| 258 |
+
|
| 259 |
+
line_patterns = [
|
| 260 |
+
r'(?m)^[^\n]*<\|ref\|>.*?<\|/ref\|><\|det\|>.*?(?:<\|/det\|>)?[^\n]*\n?',
|
| 261 |
+
r'(?m)^[^\n]*<\|det\|>.*?(?:<\|/det\|>)?[^\n]*\n?',
|
| 262 |
+
r'(?m)^[^\n]*<\|/?ref\|>[^\n]*\n?',
|
| 263 |
+
]
|
| 264 |
+
for p in line_patterns:
|
| 265 |
+
text = re.sub(p, '', text)
|
| 266 |
+
|
| 267 |
+
text = re.sub(r'<\|/?ref\|>', '', text)
|
| 268 |
+
text = re.sub(r'<\|/?det\|>', '', text)
|
| 269 |
+
return text
|
| 270 |
+
|
| 271 |
PREVIEW_CSS = """
|
| 272 |
<style>
|
| 273 |
.math-preview {
|
tests/test_clean_output.py
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import ast
|
| 2 |
+
import pathlib
|
| 3 |
+
import re
|
| 4 |
+
import unittest
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
def _load_clean_output():
|
| 8 |
+
app_path = pathlib.Path(__file__).resolve().parents[1] / "app.py"
|
| 9 |
+
source = app_path.read_text(encoding="utf-8")
|
| 10 |
+
module = ast.parse(source, filename=str(app_path))
|
| 11 |
+
|
| 12 |
+
wanted = {
|
| 13 |
+
"_strip_malformed_grounding",
|
| 14 |
+
"clean_output",
|
| 15 |
+
}
|
| 16 |
+
fn_nodes = [n for n in module.body if isinstance(n, ast.FunctionDef) and n.name in wanted]
|
| 17 |
+
fn_nodes.sort(key=lambda n: n.lineno)
|
| 18 |
+
|
| 19 |
+
test_mod = ast.Module(body=fn_nodes, type_ignores=[])
|
| 20 |
+
code = compile(test_mod, filename=str(app_path), mode="exec")
|
| 21 |
+
|
| 22 |
+
scope = {
|
| 23 |
+
"re": re,
|
| 24 |
+
}
|
| 25 |
+
exec(code, scope)
|
| 26 |
+
return scope["clean_output"]
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
class CleanOutputTests(unittest.TestCase):
|
| 30 |
+
def test_removes_truncated_grounding_artifact_line(self):
|
| 31 |
+
clean_output = _load_clean_output()
|
| 32 |
+
raw = (
|
| 33 |
+
"\\[ \\frac{18x-34}{(2x-3)^2} \\]\n"
|
| 34 |
+
"<|ref|>equation<|/ref|><|det|>[[50, 0, 450, 100]]\n"
|
| 35 |
+
)
|
| 36 |
+
|
| 37 |
+
cleaned = clean_output(raw, include_images=True)
|
| 38 |
+
self.assertIn("\\[ \\frac{18x-34}{(2x-3)^2} \\]", cleaned)
|
| 39 |
+
self.assertNotIn("<|ref|>", cleaned)
|
| 40 |
+
self.assertNotIn("<|det|>", cleaned)
|
| 41 |
+
self.assertNotIn("[[50, 0, 450, 100]]", cleaned)
|
| 42 |
+
|
| 43 |
+
def test_replaces_full_image_reference(self):
|
| 44 |
+
clean_output = _load_clean_output()
|
| 45 |
+
raw = "prefix\n<|ref|>image<|/ref|><|det|>[[0,0,100,100]]<|/det|>\nsuffix"
|
| 46 |
+
|
| 47 |
+
cleaned = clean_output(raw, include_images=True)
|
| 48 |
+
self.assertIn("**[Figure 1]**", cleaned)
|
| 49 |
+
self.assertNotIn("<|ref|>", cleaned)
|
| 50 |
+
self.assertNotIn("<|det|>", cleaned)
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
if __name__ == "__main__":
|
| 54 |
+
unittest.main()
|