| import requests |
| import json |
| import wikitextparser as wtp |
| from lxml import etree |
| from lxml import html |
|
|
| |
| |
|
|
| class MyWikiAPI: |
| WIKI_BASE_URL = "https://en.wikipedia.org/w/api.php" |
| WIKI_FEATURED_URL = "https://en.wikipedia.org/wiki/Wikipedia:Featured_article_candidates/Featured_log" |
| user_agent_headers = {"user-Agent": |
| "AgentsCourseAssignment/1.0 (https://huggingface.co/spaces/krzsam/Agents-Course-Assignment)"} |
|
|
| def __init__(self): |
| print(f"***KS*** Initializing Wiki API") |
|
|
| def __find_section_on_page__(self, page_title, section_title): |
| response = requests.get( |
| self.WIKI_BASE_URL, |
| headers=self.user_agent_headers, |
| params={ |
| 'action': 'parse', |
| 'format': 'json', |
| 'page': page_title, |
| 'prop': 'sections', |
| }).json() |
| sections = response['parse']['sections'] |
| section_ix_found = None |
| for sec in sections: |
| if sec["line"] == section_title: |
| section_ix_found = sec["index"] |
| return section_ix_found |
|
|
| def __get_page_section_content__(self, page_title, section_id, format="wikitext"): |
| response = requests.get( |
| self.WIKI_BASE_URL, |
| headers=self.user_agent_headers, |
| params={ |
| 'action': 'parse', |
| 'format': 'json', |
| 'page': page_title, |
| 'prop': format, |
| 'section': section_id |
| }).json() |
| return response["parse"][format]["*"] |
|
|
| def __get_featured_log__(self, month, year): |
| featured_url = f"{self.WIKI_FEATURED_URL}/{month}_{year}" |
| print(f"Getting content for: {featured_url}") |
| response = requests.get( |
| featured_url, |
| headers=self.user_agent_headers, |
| ).text |
| return response |
|
|
| def __process_featured_log__(self, html_content): |
| tree = html.fromstring(html_content) |
| article_heading = "mw-heading mw-heading3" |
| elements = tree.xpath(f"//div[@class='{article_heading}']") |
| element_texts = [] |
| for element in elements: |
| n1 = element.getnext() |
| n1_text = " ".join(n1.itertext()) |
|
|
| n2 = n1.getnext() |
| n2_text = " ".join(n2.itertext()) |
| element_text = f"{n1_text} {n2_text}" |
|
|
| element_texts.append(element_text) |
|
|
| return element_texts |
|
|
| def __is_int__(self, s): |
| try: |
| int(s) |
| except ValueError: |
| return False |
| else: |
| return True |
|
|
| def get_category(self, category, year): |
| ret = self.__get_category_pages__(category, year) |
| print(f"Got category: {category}\n{ret}") |
|
|
| def get_page_section(self, page_title, section_title): |
| section_id = self.__find_section_on_page__(page_title, section_title) |
| _ret = "" |
| if section_id is not None: |
| _ret = self.__get_page_section_content__(page_title, section_id) |
| |
| return _ret |
|
|
| def filter_section_and_table(self, section_content, sub_section_name, year_start, year_end): |
| parsed = wtp.parse(section_content) |
| sections = parsed.sections |
| section_found = None |
| for sec in sections: |
| if sec.title is not None and sec.title.find(sub_section_name) >= 0: |
| section_found = sec |
|
|
| print(f"Found matching subsection: {section_found.title}") |
|
|
| rows_collected = [] |
| if section_found is not None and section_found.tables is not None and len(section_found.tables) > 0: |
| table_data = section_found.tables[0].data() |
| for row in table_data: |
| if self.__is_int__(row[0]) and year_start <= int(row[0]) <= year_end: |
| rows_collected.append(row) |
|
|
| return rows_collected |
|
|
| def get_featured_articles(self, month, year): |
| |
| |
| full_html = self.__get_featured_log__(month, year) |
|
|
| ret = self.__process_featured_log__(full_html) |
|
|
| |
| return ret |
