|
|
|
|
|
|
|
|
from ultradata_math_parser.utils import * |
|
|
from ultradata_math_parser.config import * |
|
|
|
|
|
|
|
|
class TitleParser: |
|
|
def extract_by_meta(self, element: HtmlElement): |
|
|
for xpath in METAS: |
|
|
title = element.xpath(xpath) |
|
|
if title: |
|
|
return "".join(title) |
|
|
|
|
|
def extract_by_title(self, element: HtmlElement): |
|
|
return "".join(element.xpath("//title//text()")).strip() |
|
|
|
|
|
def extract_by_hs(self, element: HtmlElement): |
|
|
hs = element.xpath("//h1//text()|//h2//text()|//h3//text()") |
|
|
return hs or [] |
|
|
|
|
|
def extract_by_h(self, element: HtmlElement): |
|
|
for xpath in ["//h1", "//h2", "//h3"]: |
|
|
children = element.xpath(xpath) |
|
|
if not children: |
|
|
continue |
|
|
child = children[0] |
|
|
texts = child.xpath("./text()") |
|
|
if texts and len(texts): |
|
|
return texts[0].strip() |
|
|
|
|
|
def process(self, element: HtmlElement): |
|
|
title_extracted_by_meta = self.extract_by_meta(element) |
|
|
if title_extracted_by_meta: |
|
|
return title_extracted_by_meta |
|
|
title_extracted_by_h = self.extract_by_h(element) |
|
|
title_extracted_by_hs = self.extract_by_hs(element) |
|
|
title_extracted_by_title = self.extract_by_title(element) |
|
|
title_extracted_by_hs = sorted( |
|
|
title_extracted_by_hs, |
|
|
key=lambda x: similarity2(x, title_extracted_by_title), |
|
|
reverse=True, |
|
|
) |
|
|
if title_extracted_by_hs: |
|
|
return lcs_of_2(title_extracted_by_hs[0], title_extracted_by_title) |
|
|
|
|
|
if title_extracted_by_title: |
|
|
return title_extracted_by_title |
|
|
|
|
|
return title_extracted_by_h |
|
|
|
