import urllib.request import urllib.parse import json import re import ssl import sys import xml.etree.ElementTree as ET from html.parser import HTMLParser class MintoakHTMLParser(HTMLParser): def __init__(self): super().__init__() self.title = "" self.meta_description = "" self.meta_keywords = "" self.headings = [] self.paragraphs = [] self.links = [] self.images = [] self.current_tag = None self.in_title = False self.temp_text = [] self.current_href = None def handle_starttag(self, tag, attrs): self.current_tag = tag attrs_dict = dict(attrs) if tag == "title": self.in_title = True elif tag == "meta": name = attrs_dict.get("name", "").lower() prop = attrs_dict.get("property", "").lower() content = attrs_dict.get("content", "") if name == "description" or prop == "og:description": self.meta_description = content elif name == "keywords": self.meta_keywords = content elif tag in ["h1", "h2", "h3", "h4", "h5", "h6"]: self.temp_text = [] elif tag == "p": self.temp_text = [] elif tag == "a": self.temp_text = [] self.current_href = attrs_dict.get("href", "") elif tag == "img": src = attrs_dict.get("src", "") alt = attrs_dict.get("alt", "") if src: self.images.append({ "src": src, "alt": alt }) def handle_endtag(self, tag): if tag == "title": self.in_title = False self.title = "".join(self.temp_text).strip() self.temp_text = [] elif tag in ["h1", "h2", "h3", "h4", "h5", "h6"]: text = "".join(self.temp_text).strip() if text: self.headings.append({ "level": tag.upper(), "text": text }) self.temp_text = [] elif tag == "p": text = "".join(self.temp_text).strip() text = re.sub(r'\s+', ' ', text) if text: self.paragraphs.append(text) self.temp_text = [] elif tag == "a": text = "".join(self.temp_text).strip() if self.current_href: self.links.append({ "text": text, "url": self.current_href }) self.current_href = None self.temp_text = [] self.current_tag = None def handle_data(self, data): if self.in_title or self.current_tag in ["h1", "h2", "h3", "h4", "h5", "h6", "p", "a"]: self.temp_text.append(data) def get_sitemap_urls(sitemap_url): print(f"Fetching sitemap from {sitemap_url}...") req = urllib.request.Request( sitemap_url, headers={'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'} ) context = ssl._create_unverified_context() with urllib.request.urlopen(req, context=context) as response: xml_data = response.read() root = ET.fromstring(xml_data) namespace = {'ns': 'http://www.sitemaps.org/schemas/sitemap/0.9'} urls = [] for url_tag in root.findall('ns:url', namespace): loc = url_tag.find('ns:loc', namespace) if loc is not None: urls.append(loc.text.strip()) print(f"Discovered {len(urls)} total URLs in sitemap.") return urls def determine_category(url): parsed = urllib.parse.urlparse(url) path = parsed.path.strip("/") if not path: return "Home" elif "atmanirbhar-dukandar" in path: return "Merchant Case Study" elif "career" in path: return "Careers" elif "about-us" in path: return "About Us" elif "contact-us" in path: return "Contact Us" elif "news" in path: return "News" elif "brand-centre" in path: return "Brand Centre" else: return "General Info" from playwright.sync_api import sync_playwright def scrape_url(page, url): print(f"Scraping {url} via Playwright...") try: page.goto(url, wait_until="networkidle", timeout=60000) html_content = page.content() except Exception as e: print(f"Error fetching {url}: {e}") return None parser = MintoakHTMLParser() parser.feed(html_content) keywords_list = [k.strip() for k in parser.meta_keywords.split("\n")] if parser.meta_keywords else [] structured = { "url": url, "title": parser.title if parser.title else "Mintoak Page", "category": determine_category(url), "metadata": { "description": parser.meta_description, "keywords": keywords_list, "last_updated": "2026-06-17" }, "content_structure": { "headings": parser.headings, "paragraphs": parser.paragraphs, "links": parser.links, "images": parser.images } } return structured def main(): sitemap_url = "https://content.mintoak.com/api/sitemap/index.xml" urls = get_sitemap_urls(sitemap_url) filtered_urls = [] for url in urls: parsed_url = urllib.parse.urlparse(url) path = parsed_url.path.lower() # Check to avoid blogs and products if "/blog" in path or "/products" in path: continue filtered_urls.append(url) print(f"Filtered to {len(filtered_urls)} URLs (excluding blogs and products).") kb_data = [] print("Launching headless browser...") with sync_playwright() as p: browser = p.chromium.launch(headless=True) # Configure a desktop user-agent to avoid blocking context = browser.new_context( user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" ) page = context.new_page() for i, url in enumerate(filtered_urls, 1): print(f"[{i}/{len(filtered_urls)}] Processing...") scraped = scrape_url(page, url) if scraped: # If it's about-us, let's inject founders if they are not scraped if "about-us" in url: scraped["executive_leadership_team"] = [ {"name": "Raman Khanduja", "role": "Chief Executive Officer (CEO) and Co-founder", "linkedin": "https://www.linkedin.com/in/raman-khanduja-3abb443/"}, {"name": "Rama Tadepalli", "role": "Chief Product Officer (CPO) and Co-founder"}, {"name": "Sanjay Nazareth", "role": "Chief Operations Officer (COO) and Co-founder"}, {"name": "Kabeer Jain", "role": "Chief Technology Officer (CTO) and Co-founder"}, {"name": "Rohit Ramana", "role": "Chief Financial Officer (CFO) and Co-founder", "linkedin": "https://www.linkedin.com/in/rohit-ramana-26344a23/"}, {"name": "Nilesh Lonkar", "role": "VP of Engineering"} ] kb_data.append(scraped) browser.close() output_file = "data/mintoak/mintoak_data_1.json" print(f"Saving standard format KB to {output_file}...") with open(output_file, "w", encoding="utf-8") as f: json.dump(kb_data, f, indent=4, ensure_ascii=False) print("Scraping task completed successfully!") if __name__ == "__main__": main()