Spaces:
Running on Zero
Running on Zero
File size: 2,732 Bytes
db06ffa | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 | import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
from zsgdp.config import load_config
from zsgdp.parsers.external import MarkerParser, _read_external_markdown, _read_marker_markdown, normalize_marker_markdown
from zsgdp.schema import DocumentProfile, PageProfile
class MarkerParserTests(unittest.TestCase):
def test_normalize_marker_markdown_emits_common_schema(self):
profile = DocumentProfile(
doc_id="d1",
source_path="sample.pdf",
file_type="pdf",
page_count=1,
extension=".pdf",
pages=[PageProfile(page_num=1, digital_text_chars=20)],
)
candidate = normalize_marker_markdown(
markdown="# Report\n\n| A | B |\n| --- | --- |\n| 1 | 2 |\n\n",
profile=profile,
source_path="sample.pdf",
)
self.assertEqual(candidate.parser_name, "marker")
self.assertEqual(len(candidate.tables), 1)
self.assertEqual(len(candidate.figures), 1)
self.assertEqual(candidate.pages[0]["source_parser"], "marker")
def test_marker_parser_runs_markdown_through_normalizer(self):
profile = DocumentProfile(
doc_id="d1",
source_path="sample.pdf",
file_type="pdf",
page_count=1,
extension=".pdf",
pages=[PageProfile(page_num=1, digital_text_chars=20)],
)
with patch.object(MarkerParser, "available", return_value=True), patch(
"zsgdp.parsers.external.run_marker_to_markdown",
return_value="# Report\n\nBody.",
):
candidate = MarkerParser().parse("sample.pdf", profile, load_config())
self.assertEqual(candidate.parser_name, "marker")
self.assertEqual(candidate.elements[0].source_parser, "marker")
self.assertEqual(candidate.provenance["requested_pages"], [1])
def test_read_marker_markdown_prefers_markdown_file(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
nested = root / "sample"
nested.mkdir()
(nested / "other.md").write_text("# Other", encoding="utf-8")
(nested / "markdown.md").write_text("# Preferred", encoding="utf-8")
markdown = _read_marker_markdown(root)
self.assertEqual(markdown, "# Preferred")
def test_read_external_markdown_falls_back_to_stdout(self):
with tempfile.TemporaryDirectory() as tmp:
markdown = _read_external_markdown(Path(tmp), parser_name="mineru", stdout="# From stdout")
self.assertEqual(markdown, "# From stdout")
if __name__ == "__main__":
unittest.main()
|