Spaces:
Sleeping
Sleeping
| import asyncio | |
| import trafilatura | |
| import tiktoken | |
| _TOKENIZER = tiktoken.get_encoding("cl100k_base") | |
| def count_tokens(text): | |
| tokens = _TOKENIZER.encode(text) | |
| return len(tokens) + 1 | |
| class Parser: | |
| def __init__(self): | |
| pass | |
| async def parse(self, html: str) -> str: | |
| """ | |
| Extracts clean text from HTML asynchronously using a thread pool. | |
| """ | |
| if not html: | |
| return "" | |
| # trafilatura is synchronous/CPU-bound, so we run it in a thread | |
| content = await asyncio.to_thread( | |
| trafilatura.extract, | |
| html, | |
| output_format="markdown", | |
| favor_recall=True, | |
| include_comments=True, | |
| include_links=True, | |
| ) | |
| result = content if content else "" | |
| token = count_tokens(result) | |
| if token > 4000: | |
| tokens = _TOKENIZER.encode(result) | |
| trimmed = tokens[:4000] | |
| return _TOKENIZER.decode(trimmed) | |
| return result | |