|
|
|
|
|
|
|
|
from copy import deepcopy |
|
|
|
|
|
from ultradata_math_parser.utils import * |
|
|
from ultradata_math_parser.parsers.base_parser import BaseParser |
|
|
from ultradata_math_parser.parsers.title_parser import TitleParser |
|
|
|
|
|
|
|
|
class ArticleParser(BaseParser): |
|
|
def __init__(self) -> None: |
|
|
super().__init__() |
|
|
|
|
|
def extract(self, html="", **kwargs) -> dict: |
|
|
base_url = kwargs.get("base_url", "") |
|
|
self.process_math = kwargs.get("process_math", self.process_math) |
|
|
self.preserve_math_containers = kwargs.get("preserve_math_containers", self.preserve_math_containers) |
|
|
self.include_tables = kwargs.get("include_tables", self.include_tables) |
|
|
self.include_images = kwargs.get("include_images", self.include_images) |
|
|
html = html.replace(" ", " ").replace(" ", " ") |
|
|
tree = load_html(html) |
|
|
if tree is None: |
|
|
raise ValueError |
|
|
|
|
|
title = TitleParser().process(tree) |
|
|
|
|
|
|
|
|
base_href = tree.xpath("//base/@href") |
|
|
|
|
|
if base_href and "http" in base_href[0]: |
|
|
base_url = base_href[0] |
|
|
|
|
|
if "://blog.csdn.net/" in base_url: |
|
|
for dtree in tree.xpath('//div[@id="content_views"]//ul[@class="pre-numbering"]'): |
|
|
self.remove_node(dtree) |
|
|
|
|
|
raw_tree = deepcopy(tree) |
|
|
working_tree = deepcopy(tree) |
|
|
|
|
|
|
|
|
format_tree = self.convert_tags(working_tree, base_url=base_url) |
|
|
format_tree = self._remove_tables_from_tree(format_tree) |
|
|
format_tree = self._remove_images_from_tree(format_tree) |
|
|
|
|
|
|
|
|
normal_tree = self.clean_tags(format_tree) |
|
|
normal_tree = self._remove_tables_from_tree(normal_tree) |
|
|
normal_tree = self._remove_images_from_tree(normal_tree) |
|
|
fallback_tree = deepcopy(normal_tree) |
|
|
|
|
|
subtree, xp_num, drop_list = self.xp_1_5(normal_tree) |
|
|
if xp_num == "others": |
|
|
subtree, drop_list = self.prune_unwanted_sections(normal_tree) |
|
|
body_html = self.get_content_html(subtree, xp_num, base_url) |
|
|
|
|
|
body_html, fallback_strategy = self.apply_fallbacks( |
|
|
primary_html=body_html, |
|
|
base_url=base_url, |
|
|
normal_tree=fallback_tree, |
|
|
raw_tree=raw_tree, |
|
|
) |
|
|
|
|
|
body_html = self._strip_tables_from_html(body_html) |
|
|
body_html = self._strip_images_from_html(body_html) |
|
|
|
|
|
text_length = self._text_length_from_html(body_html) |
|
|
|
|
|
return { |
|
|
"xp_num": xp_num, |
|
|
"drop_list": drop_list, |
|
|
"html": body_html, |
|
|
"title": title, |
|
|
"base_url": base_url, |
|
|
"fallback_strategy": fallback_strategy, |
|
|
"text_length": text_length, |
|
|
} |
|
|
|
