models.py

"""
Modèles Pydantic v2 - typage strict, computed fields, validators.
"""

from __future__ import annotations

import hashlib
from datetime import datetime, timezone
from enum import Enum
from typing import Annotated, Any, Optional
from urllib.parse import urlparse

from pydantic import (
    AnyHttpUrl,
    BaseModel,
    ConfigDict,
    Field,
    computed_field,
    field_validator,
    model_validator,
)
from pydantic_settings import BaseSettings, SettingsConfigDict

# ---------------------------------------------------------------------------
# Enums
# ---------------------------------------------------------------------------


class ScrapingMethod(str, Enum):
    AUTO = "auto"
    SCRAPY = "scrapy"
    CURL_CFFI = "curl_cffi"
    CLOUDSCRAPER = "cloudscraper"
    HTTPX = "httpx"


class ExtractionMode(str, Enum):
    RAW = "raw"
    CLEAN = "clean"
    MAIN_CONTENT = "main_content"
    FULL = "full"


# ---------------------------------------------------------------------------
# Settings (pydantic-settings v2)
# ---------------------------------------------------------------------------


class Settings(BaseSettings):
    model_config = SettingsConfigDict(
        env_file=".env",
        env_file_encoding="utf-8",
        case_sensitive=False,
        extra="ignore",
    )

    port: int = 7860
    worker_id: str = "scraper-1"
    environment: str = "production"

    # Concurrency
    max_concurrent_requests: Annotated[int, Field(ge=1, le=500)] = 100
    scrapy_concurrent_requests: Annotated[int, Field(ge=1, le=64)] = 16
    scrapy_concurrent_per_domain: Annotated[int, Field(ge=1, le=32)] = 8
    scrapy_download_delay: float = 0.0  # seconds between requests per domain

    # Timeouts
    request_timeout: Annotated[int, Field(ge=5, le=120)] = 30

    # HTTP
    user_agent: str = (
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
        "AppleWebKit/537.36 (KHTML, like Gecko) "
        "Chrome/121.0.0.0 Safari/537.36"
    )
    max_retries: Annotated[int, Field(ge=1, le=5)] = 3
    retry_backoff: Annotated[float, Field(ge=0.1, le=5.0)] = 1.0
    follow_redirects: bool = True
    verify_ssl: bool = True

    # Content
    max_content_size: int = 50_000_000

    # Cache
    enable_cache: bool = True
    cache_ttl: int = 3600
    cache_max_size: int = 1000

    # Extraction
    max_links: int = 500
    max_images: int = 100


# ---------------------------------------------------------------------------
# Request models
# ---------------------------------------------------------------------------

PositiveInt = Annotated[int, Field(gt=0)]
TimeoutInt = Annotated[int, Field(ge=5, le=120)]


class CacheConfig(BaseModel):
    model_config = ConfigDict(frozen=True)

    enabled: bool = True
    ttl: Optional[PositiveInt] = None
    force_refresh: bool = False


class ExtractionConfig(BaseModel):
    model_config = ConfigDict(frozen=True)

    mode: ExtractionMode = ExtractionMode.MAIN_CONTENT
    include_metadata: bool = True
    include_links: bool = False
    include_images: bool = False
    normalize_text: bool = True
    detect_language: bool = True
    css_selectors: Optional[list[str]] = None
    xpath_selectors: Optional[list[str]] = None


class ScrapeOptions(BaseModel):
    model_config = ConfigDict(frozen=True)

    method: ScrapingMethod = ScrapingMethod.AUTO
    headers: Optional[dict[str, str]] = None
    timeout: Optional[TimeoutInt] = None
    verify_ssl: Optional[bool] = None
    follow_redirects: Optional[bool] = None
    extraction: ExtractionConfig = Field(default_factory=ExtractionConfig)
    cache: CacheConfig = Field(default_factory=CacheConfig)


class ScrapeRequest(BaseModel):
    model_config = ConfigDict(frozen=True)

    url: AnyHttpUrl
    options: ScrapeOptions = Field(default_factory=ScrapeOptions)

    @field_validator("url", mode="before")
    @classmethod
    def reject_non_http_files(cls, v: Any) -> Any:
        url_str = str(v)
        blocked = {
            ".pdf", ".zip", ".exe", ".dmg", ".pkg",
            ".jpg", ".jpeg", ".png", ".gif", ".svg", ".webp",
            ".mp4", ".avi", ".mov", ".mp3", ".wav",
            ".css", ".woff", ".woff2", ".ttf",
        }
        path = urlparse(url_str).path.lower()
        if any(path.endswith(ext) for ext in blocked):
            raise ValueError(f"Extension non scrapable : {url_str}")
        return v


# ---------------------------------------------------------------------------
# Response models
# ---------------------------------------------------------------------------


class ContentData(BaseModel):
    raw_html: Optional[str] = None
    clean_html: Optional[str] = None
    text: Optional[str] = None
    title: Optional[str] = None
    author: Optional[str] = None
    date: Optional[str] = None
    description: Optional[str] = None
    language: Optional[str] = None
    word_count: Optional[int] = None

    @computed_field  # type: ignore[misc]
    @property
    def content_hash(self) -> Optional[str]:
        if self.text:
            return hashlib.sha256(self.text.encode()).hexdigest()
        return None


class MetadataData(BaseModel):
    og_data: Optional[dict[str, str]] = None
    twitter_data: Optional[dict[str, str]] = None
    meta_tags: Optional[dict[str, str]] = None
    canonical_url: Optional[str] = None


class LinkItem(BaseModel):
    url: str
    text: str = ""
    rel: str = ""
    title: str = ""


class ImageItem(BaseModel):
    url: str
    alt: str = ""
    title: str = ""
    width: str = ""
    height: str = ""


class LinksData(BaseModel):
    internal: list[LinkItem] = Field(default_factory=list)
    external: list[LinkItem] = Field(default_factory=list)

    @computed_field  # type: ignore[misc]
    @property
    def total_count(self) -> int:
        return len(self.internal) + len(self.external)


class ImagesData(BaseModel):
    images: list[ImageItem] = Field(default_factory=list)

    @computed_field  # type: ignore[misc]
    @property
    def total_count(self) -> int:
        return len(self.images)


class PerformanceMetrics(BaseModel):
    total_time: float
    download_time: float
    parsing_time: float
    extraction_time: float
    content_size: int
    cache_hit: bool = False
    scrapy_used: bool = False

    @computed_field  # type: ignore[misc]
    @property
    def throughput_kbps(self) -> float:
        if self.download_time > 0:
            return round(self.content_size / 1024 / self.download_time, 2)
        return 0.0


class ScrapeResponse(BaseModel):
    success: bool
    worker_id: str
    url: str
    final_url: Optional[str] = None
    status_code: Optional[int] = None
    method_used: Optional[ScrapingMethod] = None
    content: Optional[ContentData] = None
    metadata: Optional[MetadataData] = None
    links: Optional[LinksData] = None
    images: Optional[ImagesData] = None
    performance: PerformanceMetrics
    error: Optional[str] = None
    timestamp: str = Field(
        default_factory=lambda: datetime.now(timezone.utc).isoformat()
    )

    @model_validator(mode="after")
    def error_requires_failure(self) -> "ScrapeResponse":
        if self.error and self.success:
            raise ValueError("Un ScrapeResponse avec error doit avoir success=False")
        return self


class HealthResponse(BaseModel):
    status: str = "healthy"
    worker_id: str
    uptime_seconds: float
    total_requests: int
    active_requests: int
    cache_size: int
    cache_hit_rate: float
    avg_response_time: float
    error_rate: float
    methods_available: list[ScrapingMethod] = Field(
        default_factory=lambda: list(ScrapingMethod)
    )

    @computed_field  # type: ignore[misc]
    @property
    def is_degraded(self) -> bool:
        return self.error_rate > 0.3 or self.status != "healthy"


# Singleton settings
settings = Settings()