server/internet_agent.py

"""Бесплатный интернет-агент с минимум 3 способами поиска"""

import re
import requests
import hashlib
from datetime import datetime, timedelta
from typing import Dict, List, Tuple, Any, Optional
from collections import Counter

class FreeInternetAgent:
    """Интернет-агент с бесплатными поисковыми системами (минимум 3 способа)"""
    
    def __init__(self, cache_ttl: int = 3600):
        self.session = requests.Session()
        self.session.headers.update({
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
        })
        self.cache: Dict[str, Tuple[Any, datetime]] = {}
        self.cache_ttl = cache_ttl
        self._has_bs4 = False
        try:
            import bs4
            self._has_bs4 = True
        except ImportError:
            pass
        
        # SearXNG инстансы
        self.searxng_instances = [
            "https://searx.be",
            "https://search.bus-hit.me",
            "https://searx.nixnet.xyz",
            "https://searx.tuxcloud.net",
            "https://searx.moe",
        ]
    
    def _get_cache_key(self, *args, **kwargs) -> str:
        key = f"{args}_{sorted(kwargs.items())}"
        return hashlib.md5(key.encode()).hexdigest()
    
    def _get_from_cache(self, key: str) -> Optional[Any]:
        if key in self.cache:
            data, timestamp = self.cache[key]
            if datetime.now() - timestamp < timedelta(seconds=self.cache_ttl):
                return data
            else:
                del self.cache[key]
        return None
    
    def _save_to_cache(self, key: str, data: Any) -> None:
        self.cache[key] = (data, datetime.now())
    
    def search_web(self, query: str, num_results: int = 5) -> List[Dict[str, str]]:
        """Умный поиск с несколькими источниками (минимум 3 способа)"""
        cache_key = self._get_cache_key('search', query, num_results)
        cached = self._get_from_cache(cache_key)
        if cached is not None:
            return cached
        
        results = []
        
        # Способ 1: SearXNG (мета-поиск)
        results = self._search_searxng(query, num_results)
        
        # Способ 2: DuckDuckGo API
        if not results:
            results = self._search_duckduckgo(query, num_results)
        
        # Способ 3: Google (парсинг)
        if not results:
            results = self._search_google(query, num_results)
        
        # Способ 4: Яндекс (для русского) — опционально
        if not results and any(ord(c) > 1024 for c in query):
            results = self._search_yandex(query, num_results)
        
        self._save_to_cache(cache_key, results)
        return results
    
    def _search_searxng(self, query: str, num_results: int) -> List[Dict[str, str]]:
        results = []
        for instance in self.searxng_instances:
            try:
                url = f"{instance}/search"
                params = {
                    "q": query,
                    "format": "json",
                    "categories": "general",
                    "engines": "google,bing,duckduckgo,startpage",
                    "language": "en",
                    "pageno": 1
                }
                response = self.session.get(url, params=params, timeout=20)
                response.raise_for_status()
                data = response.json()
                
                if 'results' in data:
                    for item in data['results'][:num_results]:
                        results.append({
                            'title': item.get('title', '')[:100],
                            'url': item.get('url', ''),
                            'snippet': item.get('content', '')[:200],
                            'source': 'searxng',
                            'engine': item.get('engine', '')
                        })
                    if results:
                        print(f"🔍 SearXNG: {len(results)} результатов")
                        break
            except Exception as e:
                continue
        return results
    
    def _search_duckduckgo(self, query: str, num_results: int) -> List[Dict[str, str]]:
        results = []
        try:
            url = f"https://api.duckduckgo.com/?q={query}&format=json&no_html=1&skip_disambig=1"
            response = self.session.get(url, timeout=15)
            data = response.json()
            
            if 'RelatedTopics' in data:
                for item in data['RelatedTopics'][:num_results]:
                    if 'Text' in item and 'FirstURL' in item:
                        results.append({
                            'title': item['Text'][:100],
                            'url': item['FirstURL'],
                            'snippet': item.get('Text', '')[:200],
                            'source': 'duckduckgo'
                        })
            print(f"🦆 DuckDuckGo: {len(results)} результатов")
        except Exception as e:
            print(f"⚠️ DuckDuckGo ошибка: {e}")
        return results
    
    def _search_google(self, query: str, num_results: int) -> List[Dict[str, str]]:
        results = []
        if not self._has_bs4:
            return results
        try:
            from bs4 import BeautifulSoup
            url = f"https://www.google.com/search?q={query}&num={num_results * 2}"
            response = self.session.get(url, timeout=20)
            soup = BeautifulSoup(response.text, 'html.parser')
            
            for g in soup.find_all('div', class_='g'):
                title_elem = g.find('h3')
                link_elem = g.find('a')
                snippet_elem = g.find('div', class_='VwiC3b')
                
                if title_elem and link_elem:
                    title = title_elem.get_text()
                    link = link_elem.get('href', '')
                    snippet = snippet_elem.get_text() if snippet_elem else ''
                    if link.startswith('/url?q='):
                        link = link.split('/url?q=')[1].split('&')[0]
                    if link.startswith('http'):
                        results.append({
                            'title': title[:100],
                            'url': link,
                            'snippet': snippet[:200],
                            'source': 'google'
                        })
                    if len(results) >= num_results:
                        break
            print(f"🔍 Google: {len(results)} результатов")
        except Exception as e:
            print(f"⚠️ Google ошибка: {e}")
        return results
    
    def _search_yandex(self, query: str, num_results: int) -> List[Dict[str, str]]:
        results = []
        if not self._has_bs4:
            return results
        try:
            from bs4 import BeautifulSoup
            url = f"https://yandex.ru/search/?text={query}&numdoc={num_results}"
            response = self.session.get(url, timeout=20)
            soup = BeautifulSoup(response.text, 'html.parser')
            
            for item in soup.find_all('li', class_='serp-item'):
                link_elem = item.find('a', class_='link')
                snippet_elem = item.find('div', class_='text-container')
                if link_elem:
                    title = link_elem.get_text()
                    link = link_elem.get('href', '')
                    snippet = snippet_elem.get_text() if snippet_elem else ''
                    if link.startswith('http'):
                        results.append({
                            'title': title[:100],
                            'url': link,
                            'snippet': snippet[:200],
                            'source': 'yandex'
                        })
                    if len(results) >= num_results:
                        break
            print(f"🔍 Яндекс: {len(results)} результатов")
        except Exception as e:
            print(f"⚠️ Яндекс ошибка: {e}")
        return results
    
    def fetch_page(self, url: str, max_size: int = 50000) -> Dict[str, Any]:
        try:
            response = self.session.get(url, timeout=30)
            response.raise_for_status()
            content = response.text[:max_size]
            return {
                "url": url,
                "status": response.status_code,
                "content": content,
                "length": len(content),
                "headers": dict(response.headers)
            }
        except Exception as e:
            return {"url": url, "error": str(e)}
    
    def analyze_website(self, url: str) -> Dict[str, Any]:
        page = self.fetch_page(url)
        if "error" in page:
            return page
        content = page.get("content", "")
        return {
            "url": url,
            "title": content.split("<title>")[1].split("</title>")[0] if "<title>" in content else "N/A",
            "has_forms": "form" in content.lower(),
            "has_scripts": "<script" in content.lower(),
            "links_count": content.count("<a "),
            "size": len(content)
        }
    
    def fetch_multiple(self, urls: List[str], max_total: int = 50000) -> List[Dict[str, Any]]:
        results = []
        total = 0
        for url in urls:
            page = self.fetch_page(url, max_size=min(10000, max_total - total))
            if "content" in page:
                total += len(page["content"])
            results.append(page)
            if total >= max_total:
                break
        return results
    
    def clear_cache(self) -> str:
        count = len(self.cache)
        self.cache.clear()
        return f"🗑️ Кэш очищен ({count} записей)"
    
    def get_cache_stats(self) -> str:
        return f"📊 Кэш: {len(self.cache)} записей, TTL: {self.cache_ttl} секунд"
    
    def _extract_keywords(self, content: str) -> List[str]:
        words = re.findall(r'\b[a-zA-Z]{4,}\b', content.lower())
        return [w for w, _ in Counter(words).most_common(10)]
    
    def _detect_language(self, content: str) -> str:
        ru_chars = len(re.findall(r'[а-яА-Я]', content))
        en_chars = len(re.findall(r'[a-zA-Z]', content))
        if ru_chars > en_chars * 0.5:
            return "ru"
        return "en"

INTERNET_AGENT = FreeInternetAgent(cache_ttl=3600)