""" custom_website_scanner.py — Custom website analysis using Requests + BeautifulSoup. Performs deep HTML analysis, form detection, link extraction, and content analysis. """ import requests from bs4 import BeautifulSoup import urllib.request, ssl from scanners.base_scanner import BaseScanner class CustomWebsiteScanner(BaseScanner): SCANNER_NAME = "Custom Website Analysis" def run(self): self.log("INFO", f"[Custom Analysis] Starting deep website analysis for {self.target}...") ctx = ssl.create_default_context() ctx.check_hostname = False ctx.verify_mode = ssl.CERT_NONE try: headers = {"User-Agent": "LarShield/2.0 (Security Audit Bot)"} if self.auth_headers: headers.update(self.auth_headers) req = urllib.request.Request(self.target, headers=headers) with urllib.request.urlopen(req, timeout=15, context=ctx) as resp: html = resp.read().decode('utf-8', errors='ignore') response_headers = {k.lower(): v for k, v in resp.getheaders()} status = resp.status self.log("SUCCESS", f"[Custom Analysis] Retrieved page: HTTP {status}") # Parse HTML with BeautifulSoup soup = BeautifulSoup(html, 'html.parser') # Perform various analyses self._analyze_forms(soup) self._analyze_links(soup) self._analyze_scripts(soup) self._analyze_meta_tags(soup) self._analyze_comments(soup) self._analyze_hidden_inputs(soup) self._analyze_external_resources(soup) self.log("SUCCESS", "[Custom Analysis] Website analysis complete.") except urllib.error.HTTPError as e: self.log("WARNING", f"[Custom Analysis] HTTP Error {e.code}: {e.reason}") except urllib.error.URLError as e: self.log("WARNING", f"[Custom Analysis] URL Error: {e.reason}") except Exception as e: self.log("WARNING", f"[Custom Analysis] Error: {e}") return self.vulns def _analyze_forms(self, soup): """Analyze HTML forms for security issues.""" self.log("INFO", "[Custom Analysis] Analyzing forms...") forms = soup.find_all('form') if not forms: self.log("INFO", "[Custom Analysis] No forms found.") return self.log("INFO", f"[Custom Analysis] Found {len(forms)} form(s).") for i, form in enumerate(forms[:10]): # Limit to first 10 forms form_action = form.get('action', '') form_method = form.get('method', 'GET').upper() self.log("INFO", f"[Custom Analysis] Form {i+1}: {form_method} -> {form_action}") # Check for insecure form method if form_method == 'GET' and form_action: self.log("WARNING", f"[Custom Analysis] Form {i+1} uses GET method for sensitive data") self.add_vuln( title="Form Uses GET Method for Sensitive Data", severity="Medium", category="Form Security", cvss_score=5.3, description=f"Form {i+1} on {self.target} uses GET method. Sensitive data in GET requests appears in browser history and server logs.", remediation="Change form method to POST for sensitive data handling." ) # Check for CSRF protection has_csrf = False for input_field in form.find_all('input'): input_name = input_field.get('name', '').lower() if 'csrf' in input_name or 'token' in input_name: has_csrf = True break if not has_csrf and form_method == 'POST': self.log("WARNING", f"[Custom Analysis] Form {i+1} may lack CSRF protection") self.add_vuln( title="Form May Lack CSRF Protection", severity="Medium", category="Form Security", cvss_score=5.9, description=f"Form {i+1} on {self.target} uses POST method but no CSRF token was detected. This could lead to Cross-Site Request Forgery attacks.", remediation="Implement CSRF tokens in all state-changing forms. Use framework-provided CSRF protection." ) def _analyze_links(self, soup): """Analyze links for security issues.""" self.log("INFO", "[Custom Analysis] Analyzing links...") links = soup.find_all('a', href=True) if not links: self.log("INFO", "[Custom Analysis] No links found.") return self.log("INFO", f"[Custom Analysis] Found {len(links)} link(s).") # Check for external links external_links = [] for link in links[:50]: # Limit to first 50 links href = link['href'] if href.startswith('http'): if self.domain not in href: external_links.append(href) if external_links: self.log("INFO", f"[Custom Analysis] Found {len(external_links)} external link(s).") # Check for suspicious external links suspicious_domains = ['bit.ly', 'tinyurl.com', 'goo.gl', 't.co', 'ow.ly'] for ext_link in external_links: for susp in suspicious_domains: if susp in ext_link: self.log("WARNING", f"[Custom Analysis] Suspicious shortener link: {ext_link}") self.add_vuln( title="Suspicious URL Shortener Link", severity="Low", category="Content Security", cvss_score=3.1, description=f"External link uses URL shortener ({susp}): {ext_link}. This could hide malicious destinations.", remediation="Avoid using URL shorteners in production. Use direct links or implement link validation." ) def _analyze_scripts(self, soup): """Analyze JavaScript files for security issues.""" self.log("INFO", "[Custom Analysis] Analyzing scripts...") scripts = soup.find_all('script') if not scripts: self.log("INFO", "[Custom Analysis] No scripts found.") return self.log("INFO", f"[Custom Analysis] Found {len(scripts)} script(s).") # Check for inline scripts inline_scripts = [s for s in scripts if s.get('src') is None] if inline_scripts: self.log("WARNING", f"[Custom Analysis] Found {len(inline_scripts)} inline script(s)") self.add_vuln( title="Inline JavaScript Detected", severity="Medium", category="Content Security", cvss_score=5.3, description=f"Found {len(inline_scripts)} inline JavaScript script(s). Inline scripts bypass Content Security Policy and increase XSS risk.", remediation="Move all JavaScript to external files and implement a strict Content Security Policy." ) # Check for external scripts external_scripts = [s.get('src') for s in scripts if s.get('src')] if external_scripts: self.log("INFO", f"[Custom Analysis] Found {len(external_scripts)} external script(s).") # Check for CDNs cdn_domains = ['cdn.jsdelivr.net', 'cdnjs.cloudflare.com', 'ajax.googleapis.com'] for script in external_scripts: for cdn in cdn_domains: if cdn in script: self.log("INFO", f"[Custom Analysis] CDN script: {script}") def _analyze_meta_tags(self, soup): """Analyze meta tags for security issues.""" self.log("INFO", "[Custom Analysis] Analyzing meta tags...") meta_tags = soup.find_all('meta') if not meta_tags: self.log("INFO", "[Custom Analysis] No meta tags found.") return self.log("INFO", f"[Custom Analysis] Found {len(meta_tags)} meta tag(s).") # Check for generator meta tag generator = soup.find('meta', attrs={'name': 'generator'}) if generator: generator_content = generator.get('content', '') self.log("WARNING", f"[Custom Analysis] Generator meta tag: {generator_content}") self.add_vuln( title="Generator Meta Tag Discloses Technology", severity="Low", category="Information Disclosure", cvss_score=3.1, description=f"Generator meta tag reveals technology: {generator_content}", remediation="Remove generator meta tag to reduce information disclosure." ) # Check for viewport meta tag viewport = soup.find('meta', attrs={'name': 'viewport'}) if not viewport: self.log("INFO", "[Custom Analysis] No viewport meta tag found (accessibility issue).") def _analyze_comments(self, soup): """Analyze HTML comments for sensitive information.""" self.log("INFO", "[Custom Analysis] Analyzing HTML comments...") comments = soup.find_all(string=lambda text: isinstance(text, str) and text.strip().startswith('