Spaces:
Sleeping
Sleeping
| from bs4 import BeautifulSoup, Tag | |
| from scanner.domain.models import TextNode | |
| def find_text_nodes(soup: BeautifulSoup) -> list[TextNode]: | |
| """Extract all text nodes from the DOM with position metadata.""" | |
| nodes = [] | |
| for el in soup.find_all(True): | |
| if not isinstance(el, Tag): | |
| continue | |
| text = el.get_text(strip=True) | |
| if not text: | |
| continue | |
| style = str(el.get("style", "") or "").lower() | |
| tag = el.name or "" | |
| is_hidden, method = _detect_hidden(style, el) | |
| selector = _make_selector(el) | |
| nodes.append( | |
| TextNode( | |
| content=text[:500], | |
| selector=selector, | |
| tag=tag, | |
| visible=not is_hidden, | |
| is_hidden=is_hidden, | |
| hidden_method=method or None, | |
| font_size=_extract_font_size(style), | |
| opacity=_extract_opacity(style) or 0.0, | |
| ) | |
| ) | |
| return nodes | |
| def get_visible_text(soup: BeautifulSoup) -> str: | |
| """Get only visible text from the DOM.""" | |
| parts = [] | |
| for el in soup.find_all(True): | |
| if not isinstance(el, Tag): | |
| continue | |
| if not _is_visible(el): | |
| continue | |
| text = el.get_text(strip=True) | |
| if text: | |
| parts.append(text) | |
| return "\n".join(parts) | |
| def _is_visible(el: Tag) -> bool: | |
| style = str(el.get("style", "") or "").lower() | |
| hidden, _ = _detect_hidden(style, el) | |
| return not hidden | |
| def _detect_hidden(style: str, el: Tag | None = None) -> tuple[bool, str | None]: | |
| if "display:none" in style or "display: none" in style: | |
| return True, "display:none" | |
| if "visibility:hidden" in style or "visibility: hidden" in style: | |
| return True, "visibility:hidden" | |
| if "opacity:0" in style or "opacity: 0" in style: | |
| return True, "opacity:0" | |
| if el and (el.get("aria-hidden") == "true" or el.get("hidden") is not None): | |
| return True, "attribute" | |
| return False, None | |
| def _make_selector(el: Tag) -> str: | |
| parts = [] | |
| if el.get("id"): | |
| return f"#{el['id']}" | |
| if el.name: | |
| parts.append(el.name) | |
| if el.get("class"): | |
| parts.append("." + ".".join(el["class"])) | |
| return "".join(parts) if parts else el.name or "unknown" | |
| def _extract_font_size(style: str) -> int | None: | |
| import re | |
| m = re.search(r"font-size\s*:\s*(\d+)", style) | |
| if m: | |
| return int(m.group(1)) | |
| return None | |
| def _extract_opacity(style: str) -> float | None: | |
| import re | |
| m = re.search(r"opacity\s*:\s*([\d.]+)", style) | |
| if m: | |
| return float(m.group(1)) | |
| return None | |