Spaces:
Sleeping
Sleeping
| from __future__ import annotations | |
| import fitz # PyMuPDF | |
| from .models import Node, SourceMeta | |
| from .ids import stable_id | |
| from .normalize import normalize_text | |
| def parse_pdf(manifest_path: str, source: SourceMeta) -> Node: | |
| # Very light heuristic: create sections when font size increases significantly | |
| import json | |
| with open(manifest_path) as f: | |
| m = json.load(f) | |
| doc = fitz.open(m["artifact_path"]) | |
| pages = doc.page_count | |
| root = Node(node_id=stable_id(source.sha256 or source.url, "PDF"), path=[], label="ROOT", text="", children=[]) | |
| current = Node(node_id=stable_id(source.sha256 or source.url, "PDF_BODY"), path=["Body"], label="Body", title="Body", text="", children=[], page_span=(1, pages)) | |
| root.children.append(current) | |
| for i in range(pages): | |
| page = doc.load_page(i) | |
| blocks = page.get_text("dict")["blocks"] | |
| for b in blocks: | |
| if "lines" not in b: continue | |
| # compute an average font size in the block | |
| sizes = [] | |
| texts = [] | |
| for l in b["lines"]: | |
| for s in l["spans"]: | |
| sizes.append(s.get("size", 0)) | |
| texts.append(s.get("text", "")) | |
| text = normalize_text(" ".join(texts)) | |
| if not text: continue | |
| avg = sum(sizes)/len(sizes) if sizes else 0 | |
| # Simple split: treat abnormally large text as a heading | |
| if avg >= 14 and len(text) <= 140: | |
| # start a new subsection | |
| sec_path = ["Body", text] | |
| node = Node(node_id=stable_id(source.sha256 or source.url, "/".join(sec_path)), | |
| path=sec_path, label=text, title=text, text="", page_span=(i+1, i+1)) | |
| current.children.append(node) | |
| else: | |
| if current.children: | |
| current.children[-1].text += (("\n\n" if current.children[-1].text else "") + text) | |
| else: | |
| current.text += (("\n\n" if current.text else "") + text) | |
| doc.close() | |
| return root |