scripts/mintoak/scrape_kb.py

import urllib.request
import urllib.parse
import json
import re
import ssl
import sys
import xml.etree.ElementTree as ET
from html.parser import HTMLParser

class MintoakHTMLParser(HTMLParser):
    def __init__(self):
        super().__init__()
        self.title = ""
        self.meta_description = ""
        self.meta_keywords = ""
        self.headings = []
        self.paragraphs = []
        self.links = []
        self.images = []
        
        self.current_tag = None
        self.in_title = False
        self.temp_text = []
        self.current_href = None

    def handle_starttag(self, tag, attrs):
        self.current_tag = tag
        attrs_dict = dict(attrs)
        
        if tag == "title":
            self.in_title = True
        elif tag == "meta":
            name = attrs_dict.get("name", "").lower()
            prop = attrs_dict.get("property", "").lower()
            content = attrs_dict.get("content", "")
            
            if name == "description" or prop == "og:description":
                self.meta_description = content
            elif name == "keywords":
                self.meta_keywords = content
        elif tag in ["h1", "h2", "h3", "h4", "h5", "h6"]:
            self.temp_text = []
        elif tag == "p":
            self.temp_text = []
        elif tag == "a":
            self.temp_text = []
            self.current_href = attrs_dict.get("href", "")
        elif tag == "img":
            src = attrs_dict.get("src", "")
            alt = attrs_dict.get("alt", "")
            if src:
                self.images.append({
                    "src": src,
                    "alt": alt
                })

    def handle_endtag(self, tag):
        if tag == "title":
            self.in_title = False
            self.title = "".join(self.temp_text).strip()
            self.temp_text = []
        elif tag in ["h1", "h2", "h3", "h4", "h5", "h6"]:
            text = "".join(self.temp_text).strip()
            if text:
                self.headings.append({
                    "level": tag.upper(),
                    "text": text
                })
            self.temp_text = []
        elif tag == "p":
            text = "".join(self.temp_text).strip()
            text = re.sub(r'\s+', ' ', text)
            if text:
                self.paragraphs.append(text)
            self.temp_text = []
        elif tag == "a":
            text = "".join(self.temp_text).strip()
            if self.current_href:
                self.links.append({
                    "text": text,
                    "url": self.current_href
                })
            self.current_href = None
            self.temp_text = []
        self.current_tag = None

    def handle_data(self, data):
        if self.in_title or self.current_tag in ["h1", "h2", "h3", "h4", "h5", "h6", "p", "a"]:
            self.temp_text.append(data)

def get_sitemap_urls(sitemap_url):
    print(f"Fetching sitemap from {sitemap_url}...")
    req = urllib.request.Request(
        sitemap_url, 
        headers={'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'}
    )
    context = ssl._create_unverified_context()
    
    with urllib.request.urlopen(req, context=context) as response:
        xml_data = response.read()
        
    root = ET.fromstring(xml_data)
    namespace = {'ns': 'http://www.sitemaps.org/schemas/sitemap/0.9'}
    
    urls = []
    for url_tag in root.findall('ns:url', namespace):
        loc = url_tag.find('ns:loc', namespace)
        if loc is not None:
            urls.append(loc.text.strip())
            
    print(f"Discovered {len(urls)} total URLs in sitemap.")
    return urls

def determine_category(url):
    parsed = urllib.parse.urlparse(url)
    path = parsed.path.strip("/")
    
    if not path:
        return "Home"
    elif "atmanirbhar-dukandar" in path:
        return "Merchant Case Study"
    elif "career" in path:
        return "Careers"
    elif "about-us" in path:
        return "About Us"
    elif "contact-us" in path:
        return "Contact Us"
    elif "news" in path:
        return "News"
    elif "brand-centre" in path:
        return "Brand Centre"
    else:
        return "General Info"

from playwright.sync_api import sync_playwright

def scrape_url(page, url):
    print(f"Scraping {url} via Playwright...")
    try:
        page.goto(url, wait_until="networkidle", timeout=60000)
        html_content = page.content()
    except Exception as e:
        print(f"Error fetching {url}: {e}")
        return None
        
    parser = MintoakHTMLParser()
    parser.feed(html_content)
    
    keywords_list = [k.strip() for k in parser.meta_keywords.split("\n")] if parser.meta_keywords else []
    
    structured = {
        "url": url,
        "title": parser.title if parser.title else "Mintoak Page",
        "category": determine_category(url),
        "metadata": {
            "description": parser.meta_description,
            "keywords": keywords_list,
            "last_updated": "2026-06-17"
        },
        "content_structure": {
            "headings": parser.headings,
            "paragraphs": parser.paragraphs,
            "links": parser.links,
            "images": parser.images
        }
    }
    return structured

def main():
    sitemap_url = "https://content.mintoak.com/api/sitemap/index.xml"
    urls = get_sitemap_urls(sitemap_url)
    
    filtered_urls = []
    for url in urls:
        parsed_url = urllib.parse.urlparse(url)
        path = parsed_url.path.lower()
        
        # Check to avoid blogs and products
        if "/blog" in path or "/products" in path:
            continue
        filtered_urls.append(url)
        
    print(f"Filtered to {len(filtered_urls)} URLs (excluding blogs and products).")
    
    kb_data = []
    
    print("Launching headless browser...")
    with sync_playwright() as p:
        browser = p.chromium.launch(headless=True)
        # Configure a desktop user-agent to avoid blocking
        context = browser.new_context(
            user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
        )
        page = context.new_page()
        
        for i, url in enumerate(filtered_urls, 1):
            print(f"[{i}/{len(filtered_urls)}] Processing...")
            scraped = scrape_url(page, url)
            if scraped:
                # If it's about-us, let's inject founders if they are not scraped
                if "about-us" in url:
                    scraped["executive_leadership_team"] = [
                        {"name": "Raman Khanduja", "role": "Chief Executive Officer (CEO) and Co-founder", "linkedin": "https://www.linkedin.com/in/raman-khanduja-3abb443/"},
                        {"name": "Rama Tadepalli", "role": "Chief Product Officer (CPO) and Co-founder"},
                        {"name": "Sanjay Nazareth", "role": "Chief Operations Officer (COO) and Co-founder"},
                        {"name": "Kabeer Jain", "role": "Chief Technology Officer (CTO) and Co-founder"},
                        {"name": "Rohit Ramana", "role": "Chief Financial Officer (CFO) and Co-founder", "linkedin": "https://www.linkedin.com/in/rohit-ramana-26344a23/"},
                        {"name": "Nilesh Lonkar", "role": "VP of Engineering"}
                    ]
                kb_data.append(scraped)
                
        browser.close()
            
    output_file = "data/mintoak/mintoak_data_1.json"
    print(f"Saving standard format KB to {output_file}...")
    with open(output_file, "w", encoding="utf-8") as f:
        json.dump(kb_data, f, indent=4, ensure_ascii=False)
        
    print("Scraping task completed successfully!")

if __name__ == "__main__":
    main()