Small bugfix

README.md

@@ -256,7 +256,7 @@ Pages have the keys:
 
 - `id` - unique id for the block.
 - `block_type` - the type of block. The possible block types can be seen in `marker/schema/__init__.py`.  As of this writing, they are ["Line", "Span", "FigureGroup", "TableGroup", "ListGroup", "PictureGroup", "Page", "Caption", "Code", "Figure", "Footnote", "Form", "Equation", "Handwriting", "TextInlineMath", "ListItem", "PageFooter", "PageHeader", "Picture", "SectionHeader", "Table", "Text", "TableOfContents", "Document"]
-- `html` - the HTML for the page.  Note that this will have recursive references to children.  The `content-ref` tags must be replaced with the child content if you want the full html.  You can see an example of this at `marker/renderers/__init__.py:BaseRender.extract_block_html`.
+- `html` - the HTML for the page.  Note that this will have recursive references to children.  The `content-ref` tags must be replaced with the child content if you want the full html.  You can see an example of this at `marker/output.py:json_to_html`.  That function will take in a single block from the json output, and turn it into HTML.
 - `polygon` - the 4-corner polygon of the page, in (x1,y1), (x2,y2), (x3, y3), (x4, y4) format.  (x1,y1) is the top left, and coordinates go clockwise.
 - `children` - the child blocks.
 

marker/output.py

@@ -1,13 +1,31 @@
 import json
 import os
 
+from bs4 import BeautifulSoup
 from pydantic import BaseModel
 
 from marker.renderers.html import HTMLOutput
-from marker.renderers.json import JSONOutput
+from marker.renderers.json import JSONOutput, JSONBlockOutput
 from marker.renderers.markdown import MarkdownOutput
 from marker.settings import settings
 
+def json_to_html(block: JSONBlockOutput):
+    # Utility function to take in json block output and give html for the block.
+    if not getattr(block, "children", None):
+        return block.html
+    else:
+        child_html = [json_to_html(child) for child in block.children]
+        child_ids = [child.id for child in block.children]
+
+        soup = BeautifulSoup(block.html, "html.parser")
+        content_refs = soup.find_all("content-ref")
+        for ref in content_refs:
+            src_id = ref.attrs["src"]
+            if src_id in child_ids:
+                child_soup = BeautifulSoup(child_html[child_ids.index(src_id)], "html.parser")
+                ref.replace_with(child_soup)
+        return str(soup)
+
 
 def output_exists(output_dir: str, fname_base: str):
     exts = ["md", "html", "json"]

marker/processors/list.py

@@ -66,6 +66,10 @@ class ListProcessor(BaseProcessor):
                 for list_item_id in block.structure:
                     list_item_block: ListItem = page.get_block(list_item_id)
 
+                    # This can be a line sometimes
+                    if list_item_block.block_type != BlockTypes.ListItem:
+                        continue
+
                     while stack and list_item_block.polygon.x_start <= stack[-1].polygon.x_start + (self.min_x_indent * page.polygon.width):
                         stack.pop()
 

pyproject.toml

@@ -28,7 +28,7 @@ texify = "^0.2.1"
 rapidfuzz = "^3.8.1"
 surya-ocr = "~0.9.3"
 regex = "^2024.4.28"
-pdftext = "~0.5.0"
+pdftext = "~0.5.1"
 markdownify = "^0.13.1"
 click = "^8.1.7"
 google-generativeai = "^0.8.3"