Spaces:
Sleeping
Sleeping
| from __future__ import annotations | |
| from bs4 import BeautifulSoup | |
| import trafilatura, json | |
| from .models import Node, SourceMeta | |
| from .ids import stable_id | |
| from .normalize import normalize_text | |
| HEADING_TAGS = ["h1","h2","h3","h4","h5","h6"] | |
| def parse_html(manifest_path: str, source: SourceMeta) -> Node: | |
| with open(manifest_path) as f: | |
| m = json.load(f) | |
| with open(m["artifact_path"], "rb") as f: | |
| html = f.read().decode("utf-8", errors="ignore") | |
| # Try trafilatura to get main content | |
| main_html = trafilatura.extract(html, include_tables=True, output_format="xml", include_comments=False) or html | |
| soup = BeautifulSoup(main_html, "lxml") | |
| root_id = stable_id(source.sha256 or source.url, "HTML") | |
| root = Node(node_id=root_id, path=[], label="ROOT", text="", html=None, children=[]) | |
| # Build outline by headings; group following siblings until the next same-or-higher level | |
| def level(tag_name:str)->int: return int(tag_name[1]) if tag_name and tag_name[0]=="h" and tag_name[1].isdigit() else 7 | |
| stack: list[tuple[int, Node]] = [(0, root)] | |
| for el in soup.find_all(HEADING_TAGS + ["p","li","table","pre"]): | |
| if el.name in HEADING_TAGS: | |
| lvl = level(el.name) | |
| title = normalize_text(el.get_text(" ", strip=True)) | |
| path = [*(n.label for _, n in stack[1:]), title] | |
| node = Node( | |
| node_id=stable_id(source.sha256 or source.url, "/".join(path)), | |
| path=path, label=title, title=title, text="", html=None, children=[] | |
| ) | |
| while stack and stack[-1][0] >= lvl: stack.pop() | |
| stack[-1][1].children.append(node) | |
| stack.append((lvl, node)) | |
| else: | |
| txt = normalize_text(el.get_text(" ", strip=True)) | |
| if not txt: continue | |
| stack[-1][1].text += (("\n\n" if stack[-1][1].text else "") + txt) | |
| # If no headings found, dump whole text under one child | |
| if not root.children: | |
| plain = normalize_text(BeautifulSoup(main_html, "lxml").get_text(" ", strip=True)) | |
| node = Node( | |
| node_id=stable_id(source.sha256 or source.url, "BODY"), | |
| path=["Body"], label="Body", title="Body", text=plain | |
| ) | |
| root.children.append(node) | |
| return root |