Spaces:
Running
Running
File size: 5,109 Bytes
9d0fd45 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 | """Detect a scrape that came back un-rendered (a JavaScript "app shell")."""
from __future__ import annotations
import html
import re
from html.parser import HTMLParser
# A reader prefixes its markdown with a ``Key: value`` block (Jina emits
# Title / URL Source / Published Time / …). Only the body below it is content.
_READER_HEADER_RE = re.compile(
r"\A(?:(?:Title|URL Source|Published Time|Content Length|Images|Links|"
r"Warning|Markdown Content):[^\n]*\n+)+",
re.IGNORECASE,
)
_HTML_DOC_RE = re.compile(
r"\A(?:\ufeff|\s|<!--.*?-->|<\?xml[^>]*>)*"
r"(?:<!doctype\s+html\b|<html\b|<head\b|<body\b)",
re.IGNORECASE | re.DOTALL,
)
_SCRIPT_TAG_RE = re.compile(r"<script\b", re.IGNORECASE)
_NOSCRIPT_JS_RE = re.compile(
r"<noscript\b[^>]*>[\s\S]*?"
r"(?:enable|requires?|turn on|activate)\s+(?:your\s+)?javascript",
re.IGNORECASE,
)
_JS_INTERSTITIAL_RE = re.compile(
r"\A(?:"
r"(?:please\s+)?(?:enable|turn on|activate)\s+(?:your\s+)?javascript"
r"|javascript\s+(?:is\s+)?(?:required|disabled)"
r"|(?:this\s+)?(?:site|page|app|application)\s+requires?\s+javascript"
r")\b",
re.IGNORECASE,
)
_EMPTY_APP_MOUNT_RE = re.compile(
r"<(?:div|main|section)\b[^>]*(?:id|class)\s*=\s*['\"][^'\"]*"
r"(?:app|root|__next|__nuxt)[^'\"]*['\"][^>]*>"
r"(?:\s|<!--[\s\S]*?-->)*</(?:div|main|section)\s*>",
re.IGNORECASE,
)
# Body shorter than this counts as "suspiciously empty — worth one retry".
# Measured: shell stubs carry 150-170 chars of body, the real page ~4800.
MIN_RENDERED_BODY_CHARS = 400
_MAX_SHELL_VISIBLE_BODY_CHARS = 80
_MAX_JS_INTERSTITIAL_CHARS = 200
class _VisibleBodyParser(HTMLParser):
"""Collect visible body text while ignoring script/style/template payloads."""
_HIDDEN = frozenset({"script", "style", "template", "noscript", "svg"})
def __init__(self) -> None:
super().__init__(convert_charrefs=True)
self.seen_body = False
self.in_body = False
self.hidden_depth = 0
self.all_text: list[str] = []
self.body_text: list[str] = []
def handle_starttag(
self,
tag: str,
attrs: list[tuple[str, str | None]],
) -> None:
name = tag.lower()
if name == "body":
self.seen_body = True
self.in_body = True
if name in self._HIDDEN:
self.hidden_depth += 1
def handle_endtag(self, tag: str) -> None:
name = tag.lower()
if name in self._HIDDEN and self.hidden_depth:
self.hidden_depth -= 1
if name == "body":
self.in_body = False
def handle_data(self, data: str) -> None:
if self.hidden_depth or not data.strip():
return
self.all_text.append(data)
if self.in_body:
self.body_text.append(data)
def visible_text(self) -> str:
parts = self.body_text if self.seen_body else self.all_text
return " ".join(" ".join(parts).split())
def _visible_body_text(content: str) -> str:
parser = _VisibleBodyParser()
try:
parser.feed(content)
parser.close()
except Exception:
return ""
return html.unescape(parser.visible_text())
def reader_body(content: str) -> str:
"""Strip the reader's ``Title:``/``URL Source:``… preamble, leaving content."""
return _READER_HEADER_RE.sub("", content or "", count=1).strip()
def unrendered_kind(content: str) -> str | None:
"""Classify a scrape that carries no page content.
Returns:
``"shell"`` — high confidence: a raw pre-hydration DOM (a reader
returns markdown on success, so raw HTML means it could not convert
the page) or an explicit "enable JavaScript" interstitial. Nothing
is recoverable by fetching the same way again.
``"empty"`` — low confidence: the body is merely suspiciously short. A
legitimately tiny page looks identical, so callers should re-try but
must not turn this into an error.
``None`` — looks like real page content.
"""
text = (content or "").strip()
if not text:
return "empty"
head = text[:4000]
is_html = bool(_HTML_DOC_RE.match(text))
visible = _visible_body_text(text) if is_html else reader_body(text)
if (
is_html
and _SCRIPT_TAG_RE.search(head)
and not visible
):
return "shell"
if (
is_html
and _SCRIPT_TAG_RE.search(head)
and len(visible) < _MAX_SHELL_VISIBLE_BODY_CHARS
and _EMPTY_APP_MOUNT_RE.search(text)
):
return "shell"
if (
is_html
and _NOSCRIPT_JS_RE.search(head)
and len(visible) < MIN_RENDERED_BODY_CHARS
):
return "shell"
if (
not is_html
and len(visible) < _MAX_JS_INTERSTITIAL_CHARS
and _JS_INTERSTITIAL_RE.search(visible)
):
return "shell"
if len(reader_body(text)) < MIN_RENDERED_BODY_CHARS:
return "empty"
return None
__all__ = ["MIN_RENDERED_BODY_CHARS", "reader_body", "unrendered_kind"]
|