scratch/crawler.py

import os
import json
import uuid
import urllib.request
import urllib.parse
from datetime import datetime
import re

# Source and destination roots
SENTI_AI_ROOT = r"c:\Users\LENOVO\Desktop\senti_ai"
SENTI_ROOT = r"c:\Users\LENOVO\Desktop\senti"

# Keywords mapped to packs and jurisdictions
CRAWL_TARGETS = {
    "sentilaw": [
        {"query": "Tax law", "jurisdiction": "Global"},
        {"query": "Value added tax", "jurisdiction": "Global"},
        {"query": "Kenya Revenue Authority", "jurisdiction": "Kenya"},
        {"query": "South African Revenue Service", "jurisdiction": "South Africa"},
        {"query": "Internal Revenue Code", "jurisdiction": "USA"},
        {"query": "HM Revenue and Customs", "jurisdiction": "UK"},
        {"query": "Financial regulation", "jurisdiction": "Global"},
        {"query": "Anti-money laundering", "jurisdiction": "Global"},
        {"query": "Basel III", "jurisdiction": "Global"},
        {"query": "Dodd–Frank Wall Street Reform and Consumer Protection Act", "jurisdiction": "USA"}
    ],
    "sentirisk": [
        {"query": "Financial risk management", "jurisdiction": "Global"},
        {"query": "Liquidity risk", "jurisdiction": "Global"},
        {"query": "Operational risk management", "jurisdiction": "Global"},
        {"query": "Concentration risk", "jurisdiction": "Global"},
        {"query": "Stress testing (finance)", "jurisdiction": "Global"},
        {"query": "Value at risk", "jurisdiction": "Global"},
        {"query": "Capital adequacy ratio", "jurisdiction": "Global"},
        {"query": "Non-performing loan", "jurisdiction": "Global"},
        {"query": "Systemic risk", "jurisdiction": "Global"},
        {"query": "Credit risk", "jurisdiction": "Global"}
    ],
    "senticredit": [
        {"query": "Credit score", "jurisdiction": "Global"},
        {"query": "Underwriting", "jurisdiction": "Global"},
        {"query": "Credit bureau", "jurisdiction": "Global"},
        {"query": "FICO", "jurisdiction": "USA"},
        {"query": "Debt-to-income ratio", "jurisdiction": "Global"},
        {"query": "Collateral (finance)", "jurisdiction": "Global"},
        {"query": "Credit default swap", "jurisdiction": "Global"},
        {"query": "Consumer credit", "jurisdiction": "Global"},
        {"query": "Mortgage underwriting", "jurisdiction": "Global"}
    ],
    "senticoach": [
        {"query": "Personal finance", "jurisdiction": "Global"},
        {"query": "Budget", "jurisdiction": "Global"},
        {"query": "Financial literacy", "jurisdiction": "Global"},
        {"query": "Retirement planning", "jurisdiction": "Global"},
        {"query": "Savings account", "jurisdiction": "Global"},
        {"query": "Debt consolidation", "jurisdiction": "Global"},
        {"query": "Investment strategy", "jurisdiction": "Global"},
        {"query": "Emergency fund", "jurisdiction": "Global"},
        {"query": "Wealth management", "jurisdiction": "Global"}
    ]
}

def clean_text(text: str) -> str:
    # Remove Wikipedia markup and markdown remnants
    text = re.sub(r'==.*?==', '', text) # Remove headers
    text = re.sub(r'\[\[(?:[^|\]]*\|)?([^\]]+)\]\]', r'\1', text) # Remove internal links markup
    text = re.sub(r'\{\{[^\}]*\}\}', '', text) # Remove templates
    text = re.sub(r'Ref[0-9]+', '', text) # Remove references
    text = re.sub(r'\s+', ' ', text) # Normalize whitespaces
    return text.strip()

def fetch_wikipedia_article(title: str) -> str:
    params = {
        "action": "query",
        "format": "json",
        "titles": title,
        "prop": "extracts",
        "explaintext": 1,
        "redirects": 1
    }
    url = "https://en.wikipedia.org/w/api.php?" + urllib.parse.urlencode(params)
    
    req = urllib.request.Request(
        url,
        headers={'User-Agent': 'SentiAICrawler/1.0 (contact: admin@sentiai.com)'}
    )
    
    try:
        with urllib.request.urlopen(req, timeout=15) as response:
            res_data = json.loads(response.read().decode('utf-8'))
            pages = res_data.get("query", {}).get("pages", {})
            for page_id, page in pages.items():
                if "extract" in page:
                    return page["extract"]
    except Exception as e:
        print(f"Error fetching page '{title}': {e}")
    return ""

def crawl_and_clean():
    print("=== STARTING SENTI AI WEB CRAWLER AND DATA INGESTION ===")
    
    total_crawled = 0
    
    for pack_name, targets in CRAWL_TARGETS.items():
        print(f"\nProcessing data pack: {pack_name}...")
        
        # Determine directories
        senti_ai_cleaned_dir = os.path.join(SENTI_AI_ROOT, pack_name, "cleaned")
        senti_cleaned_dir = os.path.join(SENTI_ROOT, pack_name, "cleaned")
        
        os.makedirs(senti_ai_cleaned_dir, exist_ok=True)
        os.makedirs(senti_cleaned_dir, exist_ok=True)
        
        for target in targets:
            query = target["query"]
            jurisdiction = target["jurisdiction"]
            
            print(f"Crawling Wikipedia for '{query}' (Jurisdiction: {jurisdiction})...")
            raw_content = fetch_wikipedia_article(query)
            
            if not raw_content or len(raw_content) < 500:
                print(f"Skipping '{query}': Insufficient content fetched.")
                continue
                
            cleaned_content = clean_text(raw_content)
            
            # Format document
            doc_id = str(uuid.uuid4())
            doc = {
                "doc_id": doc_id,
                "source": {
                    "source_id": f"crawled_wiki_{uuid.uuid4().hex[:10]}",
                    "url": f"https://en.wikipedia.org/wiki/{urllib.parse.quote(query)}",
                    "name": f"wikipedia_{query.replace(' ', '_').lower()}.txt",
                    "author": "Wikipedia Contributors",
                    "publisher": "Wikimedia Foundation",
                    "access_date": datetime.utcnow().isoformat(),
                    "license_status": "CC-BY-SA",
                    "jurisdiction": jurisdiction,
                    "language": "en",
                    "topic_tags": [query.lower(), pack_name.lower()],
                    "pack_assignment": pack_name.upper(), # Matches the original naming assignment e.g. SENTILAW
                    "file_type": "txt",
                    "checksum": uuid.uuid4().hex,
                    "version": "1.0.0",
                    "quality_score": 0.9,
                    "ingestion_timestamp": datetime.utcnow().isoformat(),
                    "extra_metadata": {
                        "crawled_via": "SentiAICrawler"
                    }
                },
                "content": {
                    "raw_text": raw_content,
                    "cleaned_text": cleaned_content
                }
            }
            
            # Write to both locations to ensure sync
            for dest_dir in [senti_ai_cleaned_dir, senti_cleaned_dir]:
                if os.path.exists(os.path.dirname(dest_dir)): # Verify parent path exists
                    dest_file = os.path.join(dest_dir, f"{doc_id}.json")
                    with open(dest_file, "w", encoding="utf-8") as f:
                        json.dump(doc, f, indent=2)
                        
            print(f"Successfully saved and cleaned '{query}' -> {doc_id}.json")
            total_crawled += 1
            
    print(f"\n=== CRAWLING COMPLETED. INGESTED AND CLEANED {total_crawled} GLOBAL DATASETS ===")

if __name__ == "__main__":
    crawl_and_clean()