""" Modèles Pydantic v2 - typage strict, computed fields, validators. """ from __future__ import annotations import hashlib from datetime import datetime, timezone from enum import Enum from typing import Annotated, Any, Optional from urllib.parse import urlparse from pydantic import ( AnyHttpUrl, BaseModel, ConfigDict, Field, computed_field, field_validator, model_validator, ) from pydantic_settings import BaseSettings, SettingsConfigDict # --------------------------------------------------------------------------- # Enums # --------------------------------------------------------------------------- class ScrapingMethod(str, Enum): AUTO = "auto" SCRAPY = "scrapy" CURL_CFFI = "curl_cffi" CLOUDSCRAPER = "cloudscraper" HTTPX = "httpx" class ExtractionMode(str, Enum): RAW = "raw" CLEAN = "clean" MAIN_CONTENT = "main_content" FULL = "full" # --------------------------------------------------------------------------- # Settings (pydantic-settings v2) # --------------------------------------------------------------------------- class Settings(BaseSettings): model_config = SettingsConfigDict( env_file=".env", env_file_encoding="utf-8", case_sensitive=False, extra="ignore", ) port: int = 7860 worker_id: str = "scraper-1" environment: str = "production" # Concurrency max_concurrent_requests: Annotated[int, Field(ge=1, le=500)] = 100 scrapy_concurrent_requests: Annotated[int, Field(ge=1, le=64)] = 16 scrapy_concurrent_per_domain: Annotated[int, Field(ge=1, le=32)] = 8 scrapy_download_delay: float = 0.0 # seconds between requests per domain # Timeouts request_timeout: Annotated[int, Field(ge=5, le=120)] = 30 # HTTP user_agent: str = ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " "AppleWebKit/537.36 (KHTML, like Gecko) " "Chrome/121.0.0.0 Safari/537.36" ) max_retries: Annotated[int, Field(ge=1, le=5)] = 3 retry_backoff: Annotated[float, Field(ge=0.1, le=5.0)] = 1.0 follow_redirects: bool = True verify_ssl: bool = True # Content max_content_size: int = 50_000_000 # Cache enable_cache: bool = True cache_ttl: int = 3600 cache_max_size: int = 1000 # Extraction max_links: int = 500 max_images: int = 100 # --------------------------------------------------------------------------- # Request models # --------------------------------------------------------------------------- PositiveInt = Annotated[int, Field(gt=0)] TimeoutInt = Annotated[int, Field(ge=5, le=120)] class CacheConfig(BaseModel): model_config = ConfigDict(frozen=True) enabled: bool = True ttl: Optional[PositiveInt] = None force_refresh: bool = False class ExtractionConfig(BaseModel): model_config = ConfigDict(frozen=True) mode: ExtractionMode = ExtractionMode.MAIN_CONTENT include_metadata: bool = True include_links: bool = False include_images: bool = False normalize_text: bool = True detect_language: bool = True css_selectors: Optional[list[str]] = None xpath_selectors: Optional[list[str]] = None class ScrapeOptions(BaseModel): model_config = ConfigDict(frozen=True) method: ScrapingMethod = ScrapingMethod.AUTO headers: Optional[dict[str, str]] = None timeout: Optional[TimeoutInt] = None verify_ssl: Optional[bool] = None follow_redirects: Optional[bool] = None extraction: ExtractionConfig = Field(default_factory=ExtractionConfig) cache: CacheConfig = Field(default_factory=CacheConfig) class ScrapeRequest(BaseModel): model_config = ConfigDict(frozen=True) url: AnyHttpUrl options: ScrapeOptions = Field(default_factory=ScrapeOptions) @field_validator("url", mode="before") @classmethod def reject_non_http_files(cls, v: Any) -> Any: url_str = str(v) blocked = { ".pdf", ".zip", ".exe", ".dmg", ".pkg", ".jpg", ".jpeg", ".png", ".gif", ".svg", ".webp", ".mp4", ".avi", ".mov", ".mp3", ".wav", ".css", ".woff", ".woff2", ".ttf", } path = urlparse(url_str).path.lower() if any(path.endswith(ext) for ext in blocked): raise ValueError(f"Extension non scrapable : {url_str}") return v # --------------------------------------------------------------------------- # Response models # --------------------------------------------------------------------------- class ContentData(BaseModel): raw_html: Optional[str] = None clean_html: Optional[str] = None text: Optional[str] = None title: Optional[str] = None author: Optional[str] = None date: Optional[str] = None description: Optional[str] = None language: Optional[str] = None word_count: Optional[int] = None @computed_field # type: ignore[misc] @property def content_hash(self) -> Optional[str]: if self.text: return hashlib.sha256(self.text.encode()).hexdigest() return None class MetadataData(BaseModel): og_data: Optional[dict[str, str]] = None twitter_data: Optional[dict[str, str]] = None meta_tags: Optional[dict[str, str]] = None canonical_url: Optional[str] = None class LinkItem(BaseModel): url: str text: str = "" rel: str = "" title: str = "" class ImageItem(BaseModel): url: str alt: str = "" title: str = "" width: str = "" height: str = "" class LinksData(BaseModel): internal: list[LinkItem] = Field(default_factory=list) external: list[LinkItem] = Field(default_factory=list) @computed_field # type: ignore[misc] @property def total_count(self) -> int: return len(self.internal) + len(self.external) class ImagesData(BaseModel): images: list[ImageItem] = Field(default_factory=list) @computed_field # type: ignore[misc] @property def total_count(self) -> int: return len(self.images) class PerformanceMetrics(BaseModel): total_time: float download_time: float parsing_time: float extraction_time: float content_size: int cache_hit: bool = False scrapy_used: bool = False @computed_field # type: ignore[misc] @property def throughput_kbps(self) -> float: if self.download_time > 0: return round(self.content_size / 1024 / self.download_time, 2) return 0.0 class ScrapeResponse(BaseModel): success: bool worker_id: str url: str final_url: Optional[str] = None status_code: Optional[int] = None method_used: Optional[ScrapingMethod] = None content: Optional[ContentData] = None metadata: Optional[MetadataData] = None links: Optional[LinksData] = None images: Optional[ImagesData] = None performance: PerformanceMetrics error: Optional[str] = None timestamp: str = Field( default_factory=lambda: datetime.now(timezone.utc).isoformat() ) @model_validator(mode="after") def error_requires_failure(self) -> "ScrapeResponse": if self.error and self.success: raise ValueError("Un ScrapeResponse avec error doit avoir success=False") return self class HealthResponse(BaseModel): status: str = "healthy" worker_id: str uptime_seconds: float total_requests: int active_requests: int cache_size: int cache_hit_rate: float avg_response_time: float error_rate: float methods_available: list[ScrapingMethod] = Field( default_factory=lambda: list(ScrapingMethod) ) @computed_field # type: ignore[misc] @property def is_degraded(self) -> bool: return self.error_rate > 0.3 or self.status != "healthy" # Singleton settings settings = Settings()