Spaces:
Sleeping
Sleeping
File size: 1,045 Bytes
34d0dc9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 | import asyncio
import trafilatura
import tiktoken
_TOKENIZER = tiktoken.get_encoding("cl100k_base")
def count_tokens(text):
tokens = _TOKENIZER.encode(text)
return len(tokens) + 1
class Parser:
def __init__(self):
pass
async def parse(self, html: str) -> str:
"""
Extracts clean text from HTML asynchronously using a thread pool.
"""
if not html:
return ""
# trafilatura is synchronous/CPU-bound, so we run it in a thread
content = await asyncio.to_thread(
trafilatura.extract,
html,
output_format="markdown",
favor_recall=True,
include_comments=True,
include_links=True,
)
result = content if content else ""
token = count_tokens(result)
if token > 4000:
tokens = _TOKENIZER.encode(result)
trimmed = tokens[:4000]
return _TOKENIZER.decode(trimmed)
return result
|