import os import asyncio import hashlib import random import httpx from playwright.async_api import async_playwright import logging # Setup logging logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') logger = logging.getLogger(__name__) # --- UNIFIED INGESTION CONFIGURATION (OPTION A) --- HF_SPACE_URL = os.environ.get("HF_SPACE_URL", "https://harshrawat18-govbridge-api.hf.space") ADMIN_SECRET = os.environ.get("ADMIN_SECRET", "") # --- STEALTH SCRIPT COLLECTION --- STEALTH_SCRIPTS = [ """ Object.defineProperty(navigator, 'webdriver', { get: () => undefined, configurable: true }); """, """ window.chrome = { app: { isInstalled: false, InstallState: { DISABLED: 'disabled', INSTALLED: 'installed', NOT_INSTALLED: 'not_installed' }, RunningState: { CANNOT_RUN: 'cannot_run', READY_TO_RUN: 'ready_to_run', RUNNING: 'running' } }, runtime: { OnInstalledReason: { CHROME_UPDATE: 'chrome_update', INSTALL: 'install', SHARED_MODULE_UPDATE: 'shared_module_update', UPDATE: 'update' }, OnRestartRequiredReason: { APP_UPDATE: 'app_update', GC_PAUSED: 'gc_paused', OS_UPDATE: 'os_update' }, PlatformArch: { ARM: 'arm', ARM64: 'arm64', MIPS: 'mips', MIPS64: 'mips64', X86_32: 'x86-32', X86_64: 'x86-64' }, PlatformNaclArch: { ARM: 'arm', MIPS: 'mips', MIPS64: 'mips64', X86_32: 'x86-32', X86_64: 'x86-64' }, PlatformOs: { ANDROID: 'android', CROS: 'cros', LINUX: 'linux', MAC: 'mac', OPENBSD: 'openbsd', WIN: 'win' }, RequestUpdateCheckStatus: { NO_UPDATE: 'no_update', THROTTLED: 'throttled', UPDATE_AVAILABLE: 'update_available' } } }; """, """ const makePlugin = (name, filename, description, mimeTypes) => { const plugin = Object.create(Plugin.prototype); Object.defineProperties(plugin, { name: { value: name, writable: false }, filename: { value: filename, writable: false }, description: { value: description, writable: false }, length: { value: mimeTypes.length } }); mimeTypes.forEach((mt, i) => { const mimeType = Object.create(MimeType.prototype); Object.defineProperties(mimeType, { type: { value: mt.type }, suffixes: { value: mt.suffixes }, description: { value: mt.description }, enabledPlugin: { value: plugin } }); plugin[i] = mimeType; }); return plugin; }; const pluginData = [ makePlugin('PDF Viewer', 'internal-pdf-viewer', 'Portable Document Format', [{ type: 'application/pdf', suffixes: 'pdf', description: '' }, { type: 'text/pdf', suffixes: 'pdf', description: '' }]), makePlugin('Chrome PDF Viewer', 'internal-pdf-viewer', 'Portable Document Format', [{ type: 'application/pdf', suffixes: 'pdf', description: '' }]) ]; const pluginArray = Object.create(PluginArray.prototype); Object.defineProperty(pluginArray, 'length', { value: pluginData.length }); pluginData.forEach((p, i) => { pluginArray[i] = p; }); pluginArray.item = (i) => pluginData[i] ?? null; pluginArray.namedItem = (name) => pluginData.find(p => p.name === name) ?? null; pluginArray.refresh = () => {}; Object.defineProperty(navigator, 'plugins', { get: () => pluginArray, configurable: true }); Object.defineProperty(navigator, 'mimeTypes', { get: () => { const arr = Object.create(MimeTypeArray.prototype); const allMimes = pluginData.flatMap(p => Array.from({length: p.length}, (_, i) => p[i])); Object.defineProperty(arr, 'length', { value: allMimes.length }); allMimes.forEach((m, i) => { arr[i] = m; }); return arr; } }); """, """ const getParameter = WebGLRenderingContext.prototype.getParameter; WebGLRenderingContext.prototype.getParameter = function(parameter) { if (parameter === 37446) return 'Intel Inc.'; if (parameter === 37445) return 'Intel Iris OpenGL Engine'; return getParameter.call(this, parameter); }; const getParameter2 = WebGL2RenderingContext.prototype.getParameter; WebGL2RenderingContext.prototype.getParameter = function(parameter) { if (parameter === 37446) return 'Intel Inc.'; if (parameter === 37445) return 'Intel Iris OpenGL Engine'; return getParameter2.call(this, parameter); }; """, """ const originalQuery = window.navigator.permissions.query; window.navigator.permissions.query = (parameters) => ( parameters.name === 'notifications' ? Promise.resolve({ state: Notification.permission }) : originalQuery(parameters) ); """, """ Object.defineProperty(navigator, 'languages', { get: () => ['en-IN', 'en-GB', 'en'], configurable: true }); """, """ const iframeProto = HTMLIFrameElement.prototype; const originalGetter = Object.getOwnPropertyDescriptor(iframeProto, 'contentWindow'); Object.defineProperty(iframeProto, 'contentWindow', { get: function() { const win = originalGetter.get.call(this); if (!win) return win; try { Object.defineProperty(win.navigator, 'webdriver', { get: () => undefined, configurable: true }); } catch(_) {} return win; } }); """ ] # --- MULTI-SITE CONFIGURATION --- GOVERNMENT_SITES = [ { "name": "MyScheme.gov.in", "url": "https://www.myscheme.gov.in/search", "source": "MyScheme Portal", "selector": "h2", "extract_type": "myscheme", "doc_type": "scheme" }, { "name": "PM-KISAN Scheme", "url": "https://pmkisan.gov.in/", "source": "Ministry of Agriculture", "selector": "h2, h3, div.content", "extract_type": "generic", "doc_type": "scheme" }, ] async def push_to_hf_api(title: str, text: str, source_url: str, ministry: str, doc_type: str): """Pushes scraped data directly to the Hugging Face Ingestion Engine""" payload = { "title": title, "text": text, "ministry": ministry, "state": "National", "source_url": source_url, "doc_type": doc_type } async with httpx.AsyncClient() as client: try: resp = await client.post( f"{HF_SPACE_URL}/api/admin/ingest", json=payload, params={"admin_key": ADMIN_SECRET}, headers={"X-Admin-Secret": ADMIN_SECRET}, timeout=30.0 ) if resp.status_code == 200: logger.info(f"✅ Ingested to Vector DB: {title[:40]}...") return True else: logger.error(f"❌ HF API Error: {resp.status_code} - {resp.text}") return False except Exception as e: logger.error(f"⚠️ Network Error pushing {title[:30]}: {e}") return False async def scrape_myscheme(page, site_config): """Specialized scraper for MyScheme portal""" try: logger.info(f"🎯 Scraping {site_config['name']}...") await asyncio.sleep(random.uniform(1.5, 3.5)) await page.goto(site_config["url"], timeout=60000, wait_until="domcontentloaded") await page.evaluate("window.scrollTo(0, document.body.scrollHeight / 3)") await asyncio.sleep(random.uniform(0.8, 1.8)) titles = await page.locator("h2").all_inner_texts() schemes = [t.strip() for t in titles if len(t.strip()) > 5 and "Filter By" not in t] results = [] if len(schemes) >= 2: for i in range(0, len(schemes) - 1, 2): try: title = schemes[i] department = schemes[i + 1] if i + 1 < len(schemes) else "Unknown" # Push directly to HF Engine success = await push_to_hf_api( title=title, text=f"Details regarding {title} provided by {department}.", source_url=site_config["url"], ministry=department, doc_type=site_config["doc_type"] ) results.append(success) except Exception as e: logger.error(f"❌ Failed to upload {schemes[i]}: {str(e)}") results.append(False) return results return [] except Exception as e: logger.error(f"❌ Error scraping {site_config['name']}: {str(e)}") return [] async def scrape_generic(page, site_config): """Generic scraper for government websites""" try: logger.info(f"🎯 Scraping {site_config['name']}...") await asyncio.sleep(random.uniform(1.5, 3.5)) await page.goto(site_config["url"], timeout=60000, wait_until="domcontentloaded") await page.evaluate("window.scrollTo(0, document.body.scrollHeight / 3)") await asyncio.sleep(random.uniform(0.8, 1.8)) selectors = site_config["selector"].split(", ") all_text = [] for selector in selectors: try: elements = await page.locator(selector).all_inner_texts() all_text.extend([t.strip() for t in elements if len(t.strip()) > 5]) except: pass results = [] if all_text: for i, text in enumerate(all_text[:10]): try: # Push directly to HF Engine success = await push_to_hf_api( title=text, text=text, source_url=site_config["url"], ministry=site_config["source"], doc_type=site_config["doc_type"] ) results.append(success) except Exception as e: logger.warning(f"⚠️ Error: {str(e)[:50]}") results.append(False) return results return [] except Exception as e: logger.error(f"❌ Error scraping {site_config['name']}: {str(e)}") return [] async def scrape_and_upload(): print("🥷 Booting up GovBridge UNIFIED Scraper (Sending to HF API)...\n") async with async_playwright() as p: # Launch hardened browser browser = await p.chromium.launch( headless=True, args=[ "--no-sandbox", "--disable-setuid-sandbox", "--disable-blink-features=AutomationControlled", "--disable-features=IsolateOrigins,site-per-process", "--disable-dev-shm-usage", "--no-first-run", "--no-default-browser-check", "--disable-infobars", ] ) context = await browser.new_context( viewport={"width": 1366, "height": 768}, user_agent=( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " "AppleWebKit/537.36 (KHTML, like Gecko) " "Chrome/124.0.0.0 Safari/537.36" ), locale="en-IN", timezone_id="Asia/Kolkata", color_scheme="light", extra_http_headers={ "Accept-Language": "en-IN,en-GB;q=0.9,en;q=0.8", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", "sec-ch-ua": '"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"', "sec-ch-ua-mobile": "?0", "sec-ch-ua-platform": '"Windows"', } ) # Inject all stealth patches for script in STEALTH_SCRIPTS: await context.add_init_script(script=script) page = await context.new_page() total_uploaded = 0 for site_config in GOVERNMENT_SITES: try: if site_config["extract_type"] == "myscheme": results = await scrape_myscheme(page, site_config) else: results = await scrape_generic(page, site_config) total_uploaded += sum(1 for r in results if r) except Exception as e: logger.error(f"❌ Critical error with {site_config['name']}: {str(e)}") await asyncio.sleep(random.uniform(2.0, 4.0)) # Delay between sites print(f"\n--- 📊 FINAL REPORT ---") print(f"✅ Total items pushed to Vector Engine: {total_uploaded}") print("\nShutting down bot.") await browser.close() if __name__ == "__main__": asyncio.run(scrape_and_upload())