Spaces:
Running
Running
| import os | |
| import asyncio | |
| import hashlib | |
| import random | |
| import httpx | |
| from playwright.async_api import async_playwright | |
| import logging | |
| # Setup logging | |
| logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') | |
| logger = logging.getLogger(__name__) | |
| # --- UNIFIED INGESTION CONFIGURATION (OPTION A) --- | |
| HF_SPACE_URL = os.environ.get("HF_SPACE_URL", "https://harshrawat18-govbridge-api.hf.space") | |
| ADMIN_SECRET = os.environ.get("ADMIN_SECRET", "") | |
| # --- STEALTH SCRIPT COLLECTION --- | |
| STEALTH_SCRIPTS = [ | |
| """ | |
| Object.defineProperty(navigator, 'webdriver', { | |
| get: () => undefined, | |
| configurable: true | |
| }); | |
| """, | |
| """ | |
| window.chrome = { | |
| app: { | |
| isInstalled: false, | |
| InstallState: { DISABLED: 'disabled', INSTALLED: 'installed', NOT_INSTALLED: 'not_installed' }, | |
| RunningState: { CANNOT_RUN: 'cannot_run', READY_TO_RUN: 'ready_to_run', RUNNING: 'running' } | |
| }, | |
| runtime: { | |
| OnInstalledReason: { CHROME_UPDATE: 'chrome_update', INSTALL: 'install', SHARED_MODULE_UPDATE: 'shared_module_update', UPDATE: 'update' }, | |
| OnRestartRequiredReason: { APP_UPDATE: 'app_update', GC_PAUSED: 'gc_paused', OS_UPDATE: 'os_update' }, | |
| PlatformArch: { ARM: 'arm', ARM64: 'arm64', MIPS: 'mips', MIPS64: 'mips64', X86_32: 'x86-32', X86_64: 'x86-64' }, | |
| PlatformNaclArch: { ARM: 'arm', MIPS: 'mips', MIPS64: 'mips64', X86_32: 'x86-32', X86_64: 'x86-64' }, | |
| PlatformOs: { ANDROID: 'android', CROS: 'cros', LINUX: 'linux', MAC: 'mac', OPENBSD: 'openbsd', WIN: 'win' }, | |
| RequestUpdateCheckStatus: { NO_UPDATE: 'no_update', THROTTLED: 'throttled', UPDATE_AVAILABLE: 'update_available' } | |
| } | |
| }; | |
| """, | |
| """ | |
| const makePlugin = (name, filename, description, mimeTypes) => { | |
| const plugin = Object.create(Plugin.prototype); | |
| Object.defineProperties(plugin, { | |
| name: { value: name, writable: false }, | |
| filename: { value: filename, writable: false }, | |
| description: { value: description, writable: false }, | |
| length: { value: mimeTypes.length } | |
| }); | |
| mimeTypes.forEach((mt, i) => { | |
| const mimeType = Object.create(MimeType.prototype); | |
| Object.defineProperties(mimeType, { | |
| type: { value: mt.type }, | |
| suffixes: { value: mt.suffixes }, | |
| description: { value: mt.description }, | |
| enabledPlugin: { value: plugin } | |
| }); | |
| plugin[i] = mimeType; | |
| }); | |
| return plugin; | |
| }; | |
| const pluginData = [ | |
| makePlugin('PDF Viewer', 'internal-pdf-viewer', 'Portable Document Format', | |
| [{ type: 'application/pdf', suffixes: 'pdf', description: '' }, | |
| { type: 'text/pdf', suffixes: 'pdf', description: '' }]), | |
| makePlugin('Chrome PDF Viewer', 'internal-pdf-viewer', 'Portable Document Format', | |
| [{ type: 'application/pdf', suffixes: 'pdf', description: '' }]) | |
| ]; | |
| const pluginArray = Object.create(PluginArray.prototype); | |
| Object.defineProperty(pluginArray, 'length', { value: pluginData.length }); | |
| pluginData.forEach((p, i) => { pluginArray[i] = p; }); | |
| pluginArray.item = (i) => pluginData[i] ?? null; | |
| pluginArray.namedItem = (name) => pluginData.find(p => p.name === name) ?? null; | |
| pluginArray.refresh = () => {}; | |
| Object.defineProperty(navigator, 'plugins', { | |
| get: () => pluginArray, | |
| configurable: true | |
| }); | |
| Object.defineProperty(navigator, 'mimeTypes', { | |
| get: () => { | |
| const arr = Object.create(MimeTypeArray.prototype); | |
| const allMimes = pluginData.flatMap(p => Array.from({length: p.length}, (_, i) => p[i])); | |
| Object.defineProperty(arr, 'length', { value: allMimes.length }); | |
| allMimes.forEach((m, i) => { arr[i] = m; }); | |
| return arr; | |
| } | |
| }); | |
| """, | |
| """ | |
| const getParameter = WebGLRenderingContext.prototype.getParameter; | |
| WebGLRenderingContext.prototype.getParameter = function(parameter) { | |
| if (parameter === 37446) return 'Intel Inc.'; | |
| if (parameter === 37445) return 'Intel Iris OpenGL Engine'; | |
| return getParameter.call(this, parameter); | |
| }; | |
| const getParameter2 = WebGL2RenderingContext.prototype.getParameter; | |
| WebGL2RenderingContext.prototype.getParameter = function(parameter) { | |
| if (parameter === 37446) return 'Intel Inc.'; | |
| if (parameter === 37445) return 'Intel Iris OpenGL Engine'; | |
| return getParameter2.call(this, parameter); | |
| }; | |
| """, | |
| """ | |
| const originalQuery = window.navigator.permissions.query; | |
| window.navigator.permissions.query = (parameters) => ( | |
| parameters.name === 'notifications' ? | |
| Promise.resolve({ state: Notification.permission }) : | |
| originalQuery(parameters) | |
| ); | |
| """, | |
| """ | |
| Object.defineProperty(navigator, 'languages', { | |
| get: () => ['en-IN', 'en-GB', 'en'], | |
| configurable: true | |
| }); | |
| """, | |
| """ | |
| const iframeProto = HTMLIFrameElement.prototype; | |
| const originalGetter = Object.getOwnPropertyDescriptor(iframeProto, 'contentWindow'); | |
| Object.defineProperty(iframeProto, 'contentWindow', { | |
| get: function() { | |
| const win = originalGetter.get.call(this); | |
| if (!win) return win; | |
| try { | |
| Object.defineProperty(win.navigator, 'webdriver', { | |
| get: () => undefined, | |
| configurable: true | |
| }); | |
| } catch(_) {} | |
| return win; | |
| } | |
| }); | |
| """ | |
| ] | |
| # --- MULTI-SITE CONFIGURATION --- | |
| GOVERNMENT_SITES = [ | |
| { | |
| "name": "MyScheme.gov.in", | |
| "url": "https://www.myscheme.gov.in/search", | |
| "source": "MyScheme Portal", | |
| "selector": "h2", | |
| "extract_type": "myscheme", | |
| "doc_type": "scheme" | |
| }, | |
| { | |
| "name": "PM-KISAN Scheme", | |
| "url": "https://pmkisan.gov.in/", | |
| "source": "Ministry of Agriculture", | |
| "selector": "h2, h3, div.content", | |
| "extract_type": "generic", | |
| "doc_type": "scheme" | |
| }, | |
| ] | |
| async def push_to_hf_api(title: str, text: str, source_url: str, ministry: str, doc_type: str): | |
| """Pushes scraped data directly to the Hugging Face Ingestion Engine""" | |
| payload = { | |
| "title": title, | |
| "text": text, | |
| "ministry": ministry, | |
| "state": "National", | |
| "source_url": source_url, | |
| "doc_type": doc_type | |
| } | |
| async with httpx.AsyncClient() as client: | |
| try: | |
| resp = await client.post( | |
| f"{HF_SPACE_URL}/api/admin/ingest", | |
| json=payload, | |
| params={"admin_key": ADMIN_SECRET}, | |
| headers={"X-Admin-Secret": ADMIN_SECRET}, | |
| timeout=30.0 | |
| ) | |
| if resp.status_code == 200: | |
| logger.info(f"✅ Ingested to Vector DB: {title[:40]}...") | |
| return True | |
| else: | |
| logger.error(f"❌ HF API Error: {resp.status_code} - {resp.text}") | |
| return False | |
| except Exception as e: | |
| logger.error(f"⚠️ Network Error pushing {title[:30]}: {e}") | |
| return False | |
| async def scrape_myscheme(page, site_config): | |
| """Specialized scraper for MyScheme portal""" | |
| try: | |
| logger.info(f"🎯 Scraping {site_config['name']}...") | |
| await asyncio.sleep(random.uniform(1.5, 3.5)) | |
| await page.goto(site_config["url"], timeout=60000, wait_until="domcontentloaded") | |
| await page.evaluate("window.scrollTo(0, document.body.scrollHeight / 3)") | |
| await asyncio.sleep(random.uniform(0.8, 1.8)) | |
| titles = await page.locator("h2").all_inner_texts() | |
| schemes = [t.strip() for t in titles if len(t.strip()) > 5 and "Filter By" not in t] | |
| results = [] | |
| if len(schemes) >= 2: | |
| for i in range(0, len(schemes) - 1, 2): | |
| try: | |
| title = schemes[i] | |
| department = schemes[i + 1] if i + 1 < len(schemes) else "Unknown" | |
| # Push directly to HF Engine | |
| success = await push_to_hf_api( | |
| title=title, | |
| text=f"Details regarding {title} provided by {department}.", | |
| source_url=site_config["url"], | |
| ministry=department, | |
| doc_type=site_config["doc_type"] | |
| ) | |
| results.append(success) | |
| except Exception as e: | |
| logger.error(f"❌ Failed to upload {schemes[i]}: {str(e)}") | |
| results.append(False) | |
| return results | |
| return [] | |
| except Exception as e: | |
| logger.error(f"❌ Error scraping {site_config['name']}: {str(e)}") | |
| return [] | |
| async def scrape_generic(page, site_config): | |
| """Generic scraper for government websites""" | |
| try: | |
| logger.info(f"🎯 Scraping {site_config['name']}...") | |
| await asyncio.sleep(random.uniform(1.5, 3.5)) | |
| await page.goto(site_config["url"], timeout=60000, wait_until="domcontentloaded") | |
| await page.evaluate("window.scrollTo(0, document.body.scrollHeight / 3)") | |
| await asyncio.sleep(random.uniform(0.8, 1.8)) | |
| selectors = site_config["selector"].split(", ") | |
| all_text = [] | |
| for selector in selectors: | |
| try: | |
| elements = await page.locator(selector).all_inner_texts() | |
| all_text.extend([t.strip() for t in elements if len(t.strip()) > 5]) | |
| except: | |
| pass | |
| results = [] | |
| if all_text: | |
| for i, text in enumerate(all_text[:10]): | |
| try: | |
| # Push directly to HF Engine | |
| success = await push_to_hf_api( | |
| title=text, | |
| text=text, | |
| source_url=site_config["url"], | |
| ministry=site_config["source"], | |
| doc_type=site_config["doc_type"] | |
| ) | |
| results.append(success) | |
| except Exception as e: | |
| logger.warning(f"⚠️ Error: {str(e)[:50]}") | |
| results.append(False) | |
| return results | |
| return [] | |
| except Exception as e: | |
| logger.error(f"❌ Error scraping {site_config['name']}: {str(e)}") | |
| return [] | |
| async def scrape_and_upload(): | |
| print("🥷 Booting up GovBridge UNIFIED Scraper (Sending to HF API)...\n") | |
| async with async_playwright() as p: | |
| # Launch hardened browser | |
| browser = await p.chromium.launch( | |
| headless=True, | |
| args=[ | |
| "--no-sandbox", | |
| "--disable-setuid-sandbox", | |
| "--disable-blink-features=AutomationControlled", | |
| "--disable-features=IsolateOrigins,site-per-process", | |
| "--disable-dev-shm-usage", | |
| "--no-first-run", | |
| "--no-default-browser-check", | |
| "--disable-infobars", | |
| ] | |
| ) | |
| context = await browser.new_context( | |
| viewport={"width": 1366, "height": 768}, | |
| user_agent=( | |
| "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " | |
| "AppleWebKit/537.36 (KHTML, like Gecko) " | |
| "Chrome/124.0.0.0 Safari/537.36" | |
| ), | |
| locale="en-IN", | |
| timezone_id="Asia/Kolkata", | |
| color_scheme="light", | |
| extra_http_headers={ | |
| "Accept-Language": "en-IN,en-GB;q=0.9,en;q=0.8", | |
| "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", | |
| "sec-ch-ua": '"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"', | |
| "sec-ch-ua-mobile": "?0", | |
| "sec-ch-ua-platform": '"Windows"', | |
| } | |
| ) | |
| # Inject all stealth patches | |
| for script in STEALTH_SCRIPTS: | |
| await context.add_init_script(script=script) | |
| page = await context.new_page() | |
| total_uploaded = 0 | |
| for site_config in GOVERNMENT_SITES: | |
| try: | |
| if site_config["extract_type"] == "myscheme": | |
| results = await scrape_myscheme(page, site_config) | |
| else: | |
| results = await scrape_generic(page, site_config) | |
| total_uploaded += sum(1 for r in results if r) | |
| except Exception as e: | |
| logger.error(f"❌ Critical error with {site_config['name']}: {str(e)}") | |
| await asyncio.sleep(random.uniform(2.0, 4.0)) # Delay between sites | |
| print(f"\n--- 📊 FINAL REPORT ---") | |
| print(f"✅ Total items pushed to Vector Engine: {total_uploaded}") | |
| print("\nShutting down bot.") | |
| await browser.close() | |
| if __name__ == "__main__": | |
| asyncio.run(scrape_and_upload()) |