Upload 6 files

Dockerfile (5).txt

@@ -0,0 +1,67 @@
+# ============================================================
+# Stage 1 : builder — compile les wheels
+# ============================================================
+FROM python:3.12-slim AS builder
+
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PIP_NO_CACHE_DIR=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    gcc g++ git curl \
+    libxml2-dev libxslt1-dev zlib1g-dev \
+    libffi-dev libssl-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /build
+COPY requirements.txt .
+RUN pip wheel --no-cache-dir --wheel-dir /wheels -r requirements.txt
+
+# ============================================================
+# Stage 2 : production — image minimale
+# ============================================================
+FROM python:3.12-slim
+
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PORT=7860 \
+    PYTHONPATH=/app
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    curl \
+    libxml2 libxslt1.1 \
+    && rm -rf /var/lib/apt/lists/* \
+    && apt-get clean
+
+# Utilisateur non-root
+RUN useradd -m -u 1000 -s /bin/bash scraper \
+    && mkdir -p /app/cache /app/logs \
+    && chown -R scraper:scraper /app
+
+WORKDIR /app
+
+# Installer les wheels compilés
+COPY --from=builder /wheels /wheels
+RUN pip install --no-cache-dir --no-index --find-links=/wheels /wheels/* \
+    && rm -rf /wheels
+
+# Copier le code applicatif
+COPY --chown=scraper:scraper app.py models.py scraper.py utils.py ./
+
+USER scraper
+
+ENV HF_HOME=/app/cache \
+    NUMEXPR_MAX_THREADS=4 \
+    OMP_NUM_THREADS=4 \
+    # Scrapy asyncio reactor — doit être défini avant tout import
+    SCRAPY_SETTINGS_MODULE=""
+
+HEALTHCHECK --interval=20s --timeout=5s --start-period=40s --retries=2 \
+    CMD curl -f -m 4 http://localhost:${PORT}/health || exit 1
+
+EXPOSE ${PORT}
+
+# Scrapy install_reactor doit être importé AVANT Twisted/Scrapy au démarrage.
+# On passe par un petit script d'amorçage pour garantir l'ordre des imports.
+CMD ["sh", "-c", "python -c 'import scrapy.utils.reactor; scrapy.utils.reactor.install_reactor(\"twisted.internet.asyncioreactor.AsyncioSelectorReactor\")' && uvicorn app:app --host 0.0.0.0 --port ${PORT} --workers 1 --loop uvloop --http httptools --log-level warning --no-access-log --limit-concurrency 100 --timeout-keep-alive 30"]

app-4.py

@@ -0,0 +1,577 @@
+"""
+Scraper SOTA v3 — FastAPI + Scrapy + Pydantic v2.
+
+Architecture :
+  - FastAPI gère les routes HTTP et la validation (Pydantic v2)
+  - Scrapy est le moteur de crawl primaire (retry, throttle, middlewares intégrés)
+  - curl_cffi / cloudscraper / httpx sont des fallbacks pour les sites protégés
+  - structlog + prometheus pour l'observabilité
+  - Cache LRU en mémoire (remplace le dict manuel)
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import time
+from collections import OrderedDict
+from contextlib import asynccontextmanager
+from datetime import datetime, timezone
+from typing import Any, Optional
+from urllib.parse import urlparse
+
+import httpx
+import orjson
+import structlog
+from curl_cffi import requests as curl_requests
+import cloudscraper
+from fastapi import BackgroundTasks, FastAPI, HTTPException, Request
+from fastapi.responses import ORJSONResponse, Response
+from prometheus_client import (
+    CONTENT_TYPE_LATEST,
+    Counter,
+    Gauge,
+    Histogram,
+    generate_latest,
+)
+from tenacity import (
+    retry,
+    retry_if_exception_type,
+    stop_after_attempt,
+    wait_exponential,
+)
+
+from models import (
+    ContentData,
+    ExtractionConfig,
+    ExtractionMode,
+    HealthResponse,
+    ImagesData,
+    ImageItem,
+    LinksData,
+    LinkItem,
+    MetadataData,
+    PerformanceMetrics,
+    ScrapingMethod,
+    ScrapeOptions,
+    ScrapeRequest,
+    ScrapeResponse,
+    settings,
+)
+from scraper import PageItem, ScrapyRunner, scrapy_runner
+from utils import ContentCleaner, URLValidator
+
+# ---------------------------------------------------------------------------
+# Logging structuré
+# ---------------------------------------------------------------------------
+
+structlog.configure(
+    processors=[
+        structlog.stdlib.filter_by_level,
+        structlog.processors.TimeStamper(fmt="iso"),
+        structlog.stdlib.add_logger_name,
+        structlog.stdlib.add_log_level,
+        structlog.processors.StackInfoRenderer(),
+        structlog.processors.format_exc_info,
+        structlog.processors.JSONRenderer(serializer=orjson.dumps),
+    ],
+    wrapper_class=structlog.stdlib.BoundLogger,
+    logger_factory=structlog.stdlib.LoggerFactory(),
+    cache_logger_on_first_use=True,
+)
+log = structlog.get_logger()
+
+# ---------------------------------------------------------------------------
+# Métriques Prometheus
+# ---------------------------------------------------------------------------
+
+REQUESTS_TOTAL = Counter("scraper_requests_total", "Total requests", ["method", "status"])
+REQUESTS_DURATION = Histogram("scraper_duration_seconds", "Request duration", ["method"])
+ACTIVE_REQUESTS = Gauge("scraper_active_requests", "Active requests")
+CACHE_HITS = Counter("scraper_cache_hits_total", "Cache hits")
+CACHE_MISSES = Counter("scraper_cache_misses_total", "Cache misses")
+ERRORS_TOTAL = Counter("scraper_errors_total", "Errors", ["error_type"])
+
+# ---------------------------------------------------------------------------
+# Cache LRU en mémoire
+# ---------------------------------------------------------------------------
+
+
+class LRUCache:
+    """Cache LRU thread-safe (asyncio) avec TTL par entrée."""
+
+    def __init__(self, max_size: int, default_ttl: int) -> None:
+        self._store: OrderedDict[str, tuple[Any, float]] = OrderedDict()
+        self.max_size = max_size
+        self.default_ttl = default_ttl
+        self._hits = 0
+        self._misses = 0
+
+    def get(self, key: str) -> Optional[Any]:
+        if key not in self._store:
+            self._misses += 1
+            CACHE_MISSES.inc()
+            return None
+        value, expires_at = self._store[key]
+        if time.monotonic() > expires_at:
+            del self._store[key]
+            self._misses += 1
+            CACHE_MISSES.inc()
+            return None
+        self._store.move_to_end(key)
+        self._hits += 1
+        CACHE_HITS.inc()
+        return value
+
+    def set(self, key: str, value: Any, ttl: Optional[int] = None) -> None:
+        effective_ttl = ttl if ttl is not None else self.default_ttl
+        if key in self._store:
+            self._store.move_to_end(key)
+        elif len(self._store) >= self.max_size:
+            self._store.popitem(last=False)  # évicte le plus ancien
+        self._store[key] = (value, time.monotonic() + effective_ttl)
+
+    @property
+    def size(self) -> int:
+        return len(self._store)
+
+    @property
+    def hit_rate(self) -> float:
+        total = self._hits + self._misses
+        return self._hits / total if total else 0.0
+
+
+# ---------------------------------------------------------------------------
+# État global du worker
+# ---------------------------------------------------------------------------
+
+
+class WorkerState:
+    def __init__(self) -> None:
+        self.start_time = time.monotonic()
+        self.total_requests = 0
+        self.active_requests = 0
+        self.total_errors = 0
+
+        self.cache = LRUCache(
+            max_size=settings.cache_max_size,
+            default_ttl=settings.cache_ttl,
+        )
+
+        self._response_times: list[float] = []
+        self._rt_max = 1000  # fenêtre glissante
+
+        # HTTP clients alternatifs (fallback)
+        self._cloudscraper = cloudscraper.create_scraper(
+            browser={"browser": "chrome", "platform": "windows", "mobile": False},
+            delay=10,
+        )
+        self._httpx_client: Optional[httpx.AsyncClient] = None
+
+    async def get_httpx_client(self) -> httpx.AsyncClient:
+        if self._httpx_client is None:
+            limits = httpx.Limits(
+                max_connections=settings.max_concurrent_requests,
+                max_keepalive_connections=settings.max_concurrent_requests // 2,
+                keepalive_expiry=30,
+            )
+            self._httpx_client = httpx.AsyncClient(
+                timeout=httpx.Timeout(settings.request_timeout),
+                limits=limits,
+                follow_redirects=settings.follow_redirects,
+                http2=True,
+                verify=settings.verify_ssl,
+            )
+        return self._httpx_client
+
+    def record_response_time(self, duration: float) -> None:
+        self._response_times.append(duration)
+        if len(self._response_times) > self._rt_max:
+            self._response_times.pop(0)
+
+    @property
+    def avg_response_time(self) -> float:
+        if not self._response_times:
+            return 0.0
+        return sum(self._response_times) / len(self._response_times)
+
+    @property
+    def error_rate(self) -> float:
+        if self.total_requests == 0:
+            return 0.0
+        return self.total_errors / self.total_requests
+
+    async def close(self) -> None:
+        if self._httpx_client:
+            await self._httpx_client.aclose()
+        await scrapy_runner.shutdown()
+
+
+state = WorkerState()
+
+# ---------------------------------------------------------------------------
+# Lifecycle FastAPI
+# ---------------------------------------------------------------------------
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    log.info(
+        "worker_startup",
+        worker_id=settings.worker_id,
+        environment=settings.environment,
+    )
+    yield
+    log.info("worker_shutdown", worker_id=settings.worker_id)
+    await state.close()
+
+
+# ---------------------------------------------------------------------------
+# Application
+# ---------------------------------------------------------------------------
+
+app = FastAPI(
+    title="Scraper SOTA v3",
+    description="Worker de scraping haute performance — FastAPI + Scrapy + Pydantic v2",
+    version="3.0.0",
+    lifespan=lifespan,
+    default_response_class=ORJSONResponse,
+)
+
+# ---------------------------------------------------------------------------
+# Middleware
+# ---------------------------------------------------------------------------
+
+
+@app.middleware("http")
+async def timing_middleware(request: Request, call_next):
+    t0 = time.perf_counter()
+    state.active_requests += 1
+    ACTIVE_REQUESTS.set(state.active_requests)
+    try:
+        response = await call_next(request)
+        elapsed = time.perf_counter() - t0
+        response.headers["X-Process-Time"] = f"{elapsed:.4f}"
+        response.headers["X-Worker-ID"] = settings.worker_id
+        state.record_response_time(elapsed)
+        return response
+    finally:
+        state.active_requests -= 1
+        ACTIVE_REQUESTS.set(state.active_requests)
+
+
+# ---------------------------------------------------------------------------
+# Routes
+# ---------------------------------------------------------------------------
+
+
+@app.get("/")
+async def root():
+    return {
+        "service": "Scraper SOTA",
+        "version": "3.0.0",
+        "worker_id": settings.worker_id,
+        "status": "operational",
+        "engine": "Scrapy + FastAPI + Pydantic v2",
+        "features": [
+            "scrapy-primary-crawler",
+            "curl_cffi / cloudscraper fallback",
+            "pydantic-v2-strict-models",
+            "lru-cache",
+            "prometheus-metrics",
+            "structured-logging",
+            "multi-method-extraction",
+        ],
+    }
+
+
+@app.get("/health", response_model=HealthResponse)
+async def health() -> HealthResponse:
+    return HealthResponse(
+        status="healthy" if state.error_rate < 0.3 else "degraded",
+        worker_id=settings.worker_id,
+        uptime_seconds=time.monotonic() - state.start_time,
+        total_requests=state.total_requests,
+        active_requests=state.active_requests,
+        cache_size=state.cache.size,
+        cache_hit_rate=state.cache.hit_rate,
+        avg_response_time=state.avg_response_time,
+        error_rate=state.error_rate,
+    )
+
+
+@app.get("/metrics")
+async def metrics():
+    return Response(content=generate_latest(), media_type=CONTENT_TYPE_LATEST)
+
+
+@app.post("/scrape", response_model=ScrapeResponse)
+async def scrape_url(
+    request: ScrapeRequest,
+    background_tasks: BackgroundTasks,
+) -> ScrapeResponse:
+    t0 = time.perf_counter()
+    url_str = str(request.url)
+    state.total_requests += 1
+
+    log.info("scrape_start", url=url_str, method=request.options.method.value)
+
+    # --- Cache ---
+    if request.options.cache.enabled and not request.options.cache.force_refresh:
+        cache_key = ContentCleaner.compute_content_hash(url_str)
+        cached = state.cache.get(cache_key)
+        if cached is not None:
+            log.info("cache_hit", url=url_str)
+            cached["performance"]["cache_hit"] = True
+            cached["performance"]["total_time"] = time.perf_counter() - t0
+            return ScrapeResponse(**cached)
+
+    try:
+        # --- Sélection de méthode ---
+        method = _select_method(request)
+
+        # --- Téléchargement ---
+        dl_start = time.perf_counter()
+        html, status_code, final_url = await _download(url_str, method, request.options)
+        dl_time = time.perf_counter() - dl_start
+
+        REQUESTS_DURATION.labels(method=method.value).observe(dl_time)
+
+        # --- Extraction ---
+        parse_start = time.perf_counter()
+        content_data = _extract_content(html, final_url, request.options.extraction)
+        parse_time = time.perf_counter() - parse_start
+
+        ex_start = time.perf_counter()
+        metadata_data = (
+            _extract_metadata(html) if request.options.extraction.include_metadata else None
+        )
+        links_data = (
+            _extract_links(html, final_url) if request.options.extraction.include_links else None
+        )
+        images_data = (
+            _extract_images(html, final_url) if request.options.extraction.include_images else None
+        )
+        ex_time = time.perf_counter() - ex_start
+
+        total_time = time.perf_counter() - t0
+
+        performance = PerformanceMetrics(
+            total_time=round(total_time, 4),
+            download_time=round(dl_time, 4),
+            parsing_time=round(parse_time, 4),
+            extraction_time=round(ex_time, 4),
+            content_size=len(html.encode("utf-8", errors="replace")),
+            cache_hit=False,
+            scrapy_used=(method == ScrapingMethod.SCRAPY),
+        )
+
+        response_dict: dict[str, Any] = {
+            "success": True,
+            "worker_id": settings.worker_id,
+            "url": url_str,
+            "final_url": final_url,
+            "status_code": status_code,
+            "method_used": method,
+            "content": content_data.model_dump(),
+            "metadata": metadata_data.model_dump() if metadata_data else None,
+            "links": links_data.model_dump() if links_data else None,
+            "images": images_data.model_dump() if images_data else None,
+            "performance": performance.model_dump(),
+            "timestamp": datetime.now(timezone.utc).isoformat(),
+        }
+
+        # --- Mise en cache ---
+        if request.options.cache.enabled:
+            cache_key = ContentCleaner.compute_content_hash(url_str)
+            state.cache.set(response_dict, cache_key, request.options.cache.ttl)
+
+        REQUESTS_TOTAL.labels(method=method.value, status="success").inc()
+        log.info("scrape_success", url=url_str, duration=total_time, method=method.value)
+
+        return ScrapeResponse(**response_dict)
+
+    except Exception as exc:
+        state.total_errors += 1
+        ERRORS_TOTAL.labels(error_type=type(exc).__name__).inc()
+        REQUESTS_TOTAL.labels(method="unknown", status="error").inc()
+        log.error("scrape_error", url=url_str, error=str(exc))
+
+        return ScrapeResponse(
+            success=False,
+            worker_id=settings.worker_id,
+            url=url_str,
+            error=str(exc),
+            performance=PerformanceMetrics(
+                total_time=round(time.perf_counter() - t0, 4),
+                download_time=0.0,
+                parsing_time=0.0,
+                extraction_time=0.0,
+                content_size=0,
+                cache_hit=False,
+            ),
+        )
+
+
+# ---------------------------------------------------------------------------
+# Logique métier
+# ---------------------------------------------------------------------------
+
+
+def _select_method(request: ScrapeRequest) -> ScrapingMethod:
+    if request.options.method != ScrapingMethod.AUTO:
+        return request.options.method
+
+    url_lower = str(request.url).lower()
+    if any(x in url_lower for x in ("cloudflare", "cf-", "captcha")):
+        return ScrapingMethod.CLOUDSCRAPER
+
+    # Scrapy est le moteur par défaut
+    return ScrapingMethod.SCRAPY
+
+
+async def _download(
+    url: str,
+    method: ScrapingMethod,
+    options: ScrapeOptions,
+) -> tuple[str, Optional[int], str]:
+    """Délègue le téléchargement à Scrapy ou aux clients HTTP alternatifs."""
+    timeout = options.timeout or settings.request_timeout
+    verify = options.verify_ssl if options.verify_ssl is not None else settings.verify_ssl
+    headers = dict(options.headers or {})
+
+    if method == ScrapingMethod.SCRAPY:
+        item: PageItem = await scrapy_runner.fetch(
+            url,
+            timeout=timeout,
+            verify_ssl=verify,
+            custom_headers=headers,
+        )
+        if item.get("error"):
+            raise RuntimeError(f"Scrapy error: {item['error']}")
+        return item["html"], item.get("status_code"), item.get("final_url", url)
+
+    # Fallbacks
+    if "User-Agent" not in headers:
+        headers["User-Agent"] = settings.user_agent
+    headers.update(
+        {
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+            "Accept-Language": "en-US,en;q=0.9",
+            "Accept-Encoding": "gzip, deflate, br",
+        }
+    )
+
+    if method == ScrapingMethod.CURL_CFFI:
+        loop = asyncio.get_running_loop()
+        resp = await loop.run_in_executor(
+            None,
+            lambda: curl_requests.get(
+                url,
+                headers=headers,
+                timeout=timeout,
+                impersonate="chrome120",
+                verify=verify,
+                allow_redirects=options.follow_redirects if options.follow_redirects is not None else settings.follow_redirects,
+            ),
+        )
+        return resp.text, resp.status_code, str(resp.url)
+
+    if method == ScrapingMethod.CLOUDSCRAPER:
+        loop = asyncio.get_running_loop()
+        resp = await loop.run_in_executor(
+            None,
+            lambda: state._cloudscraper.get(
+                url,
+                headers=headers,
+                timeout=timeout,
+                verify=verify,
+            ),
+        )
+        resp.raise_for_status()
+        return resp.text, resp.status_code, resp.url
+
+    if method == ScrapingMethod.HTTPX:
+        client = await state.get_httpx_client()
+        resp = await client.get(url, headers=headers, timeout=timeout)
+        resp.raise_for_status()
+        return resp.text, resp.status_code, str(resp.url)
+
+    raise ValueError(f"Méthode non supportée : {method}")
+
+
+def _extract_content(html: str, url: str, config: ExtractionConfig) -> ContentData:
+    if config.mode == ExtractionMode.RAW:
+        return ContentData(raw_html=html[: settings.max_content_size])
+
+    raw_extracted: dict[str, Any] = {}
+
+    if config.mode in (ExtractionMode.CLEAN, ExtractionMode.FULL):
+        clean_html = ContentCleaner.clean_html_fast(html)
+
+    if config.mode in (ExtractionMode.MAIN_CONTENT, ExtractionMode.FULL):
+        raw_extracted = ContentCleaner.extract_main_content(html, url)
+        text = raw_extracted.get("text", "")
+        if config.normalize_text:
+            text = ContentCleaner.normalize_text(text)
+        return ContentData(
+            clean_html=(ContentCleaner.clean_html_fast(html)[: settings.max_content_size]
+                        if config.mode == ExtractionMode.FULL else None),
+            text=text[: settings.max_content_size],
+            title=raw_extracted.get("title"),
+            author=raw_extracted.get("author"),
+            date=raw_extracted.get("date"),
+            description=raw_extracted.get("description"),
+            language=raw_extracted.get("language"),
+            word_count=len(text.split()),
+        )
+
+    # CLEAN only
+    return ContentData(clean_html=ContentCleaner.clean_html_fast(html)[: settings.max_content_size])
+
+
+def _extract_metadata(html: str) -> MetadataData:
+    md = ContentCleaner.extract_metadata(html)
+    return MetadataData(
+        og_data={k.replace("og_", ""): v for k, v in md.items() if k.startswith("og_")} or None,
+        twitter_data={k.replace("twitter_", ""): v for k, v in md.items() if k.startswith("twitter_")} or None,
+        meta_tags={k: v for k, v in md.items() if not k.startswith(("og_", "twitter_"))} or None,
+        canonical_url=md.get("canonical"),
+    )
+
+
+def _extract_links(html: str, base_url: str) -> LinksData:
+    base_domain = urlparse(base_url).netloc
+    all_links = ContentCleaner.extract_links(html, base_url)[: settings.max_links]
+    internal, external = [], []
+    for lk in all_links:
+        item = LinkItem(**lk)
+        if urlparse(lk["url"]).netloc == base_domain:
+            internal.append(item)
+        else:
+            external.append(item)
+    return LinksData(internal=internal, external=external)
+
+
+def _extract_images(html: str, base_url: str) -> ImagesData:
+    imgs = ContentCleaner.extract_images(html, base_url)[: settings.max_images]
+    return ImagesData(images=[ImageItem(**i) for i in imgs])
+
+
+# ---------------------------------------------------------------------------
+# Entrypoint
+# ---------------------------------------------------------------------------
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run(
+        "app:app",
+        host="0.0.0.0",
+        port=settings.port,
+        log_level="warning",
+        access_log=False,
+        loop="uvloop",
+        http="httptools",
+        limit_concurrency=settings.max_concurrent_requests,
+    )

models-3.py

@@ -0,0 +1,284 @@
+"""
+Modèles Pydantic v2 - typage strict, computed fields, validators.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from datetime import datetime, timezone
+from enum import Enum
+from typing import Annotated, Any, Optional
+from urllib.parse import urlparse
+
+from pydantic import (
+    AnyHttpUrl,
+    BaseModel,
+    ConfigDict,
+    Field,
+    computed_field,
+    field_validator,
+    model_validator,
+)
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+# ---------------------------------------------------------------------------
+# Enums
+# ---------------------------------------------------------------------------
+
+
+class ScrapingMethod(str, Enum):
+    AUTO = "auto"
+    SCRAPY = "scrapy"
+    CURL_CFFI = "curl_cffi"
+    CLOUDSCRAPER = "cloudscraper"
+    HTTPX = "httpx"
+
+
+class ExtractionMode(str, Enum):
+    RAW = "raw"
+    CLEAN = "clean"
+    MAIN_CONTENT = "main_content"
+    FULL = "full"
+
+
+# ---------------------------------------------------------------------------
+# Settings (pydantic-settings v2)
+# ---------------------------------------------------------------------------
+
+
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(
+        env_file=".env",
+        env_file_encoding="utf-8",
+        case_sensitive=False,
+        extra="ignore",
+    )
+
+    port: int = 7860
+    worker_id: str = "scraper-1"
+    environment: str = "production"
+
+    # Concurrency
+    max_concurrent_requests: Annotated[int, Field(ge=1, le=500)] = 100
+    scrapy_concurrent_requests: Annotated[int, Field(ge=1, le=64)] = 16
+    scrapy_concurrent_per_domain: Annotated[int, Field(ge=1, le=32)] = 8
+    scrapy_download_delay: float = 0.0  # seconds between requests per domain
+
+    # Timeouts
+    request_timeout: Annotated[int, Field(ge=5, le=120)] = 30
+
+    # HTTP
+    user_agent: str = (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/121.0.0.0 Safari/537.36"
+    )
+    max_retries: Annotated[int, Field(ge=1, le=5)] = 3
+    retry_backoff: Annotated[float, Field(ge=0.1, le=5.0)] = 1.0
+    follow_redirects: bool = True
+    verify_ssl: bool = True
+
+    # Content
+    max_content_size: int = 50_000_000
+
+    # Cache
+    enable_cache: bool = True
+    cache_ttl: int = 3600
+    cache_max_size: int = 1000
+
+    # Extraction
+    max_links: int = 500
+    max_images: int = 100
+
+
+# ---------------------------------------------------------------------------
+# Request models
+# ---------------------------------------------------------------------------
+
+PositiveInt = Annotated[int, Field(gt=0)]
+TimeoutInt = Annotated[int, Field(ge=5, le=120)]
+
+
+class CacheConfig(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    enabled: bool = True
+    ttl: Optional[PositiveInt] = None
+    force_refresh: bool = False
+
+
+class ExtractionConfig(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    mode: ExtractionMode = ExtractionMode.MAIN_CONTENT
+    include_metadata: bool = True
+    include_links: bool = False
+    include_images: bool = False
+    normalize_text: bool = True
+    detect_language: bool = True
+    css_selectors: Optional[list[str]] = None
+    xpath_selectors: Optional[list[str]] = None
+
+
+class ScrapeOptions(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    method: ScrapingMethod = ScrapingMethod.AUTO
+    headers: Optional[dict[str, str]] = None
+    timeout: Optional[TimeoutInt] = None
+    verify_ssl: Optional[bool] = None
+    follow_redirects: Optional[bool] = None
+    extraction: ExtractionConfig = Field(default_factory=ExtractionConfig)
+    cache: CacheConfig = Field(default_factory=CacheConfig)
+
+
+class ScrapeRequest(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    url: AnyHttpUrl
+    options: ScrapeOptions = Field(default_factory=ScrapeOptions)
+
+    @field_validator("url", mode="before")
+    @classmethod
+    def reject_non_http_files(cls, v: Any) -> Any:
+        url_str = str(v)
+        blocked = {
+            ".pdf", ".zip", ".exe", ".dmg", ".pkg",
+            ".jpg", ".jpeg", ".png", ".gif", ".svg", ".webp",
+            ".mp4", ".avi", ".mov", ".mp3", ".wav",
+            ".css", ".woff", ".woff2", ".ttf",
+        }
+        path = urlparse(url_str).path.lower()
+        if any(path.endswith(ext) for ext in blocked):
+            raise ValueError(f"Extension non scrapable : {url_str}")
+        return v
+
+
+# ---------------------------------------------------------------------------
+# Response models
+# ---------------------------------------------------------------------------
+
+
+class ContentData(BaseModel):
+    raw_html: Optional[str] = None
+    clean_html: Optional[str] = None
+    text: Optional[str] = None
+    title: Optional[str] = None
+    author: Optional[str] = None
+    date: Optional[str] = None
+    description: Optional[str] = None
+    language: Optional[str] = None
+    word_count: Optional[int] = None
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def content_hash(self) -> Optional[str]:
+        if self.text:
+            return hashlib.sha256(self.text.encode()).hexdigest()
+        return None
+
+
+class MetadataData(BaseModel):
+    og_data: Optional[dict[str, str]] = None
+    twitter_data: Optional[dict[str, str]] = None
+    meta_tags: Optional[dict[str, str]] = None
+    canonical_url: Optional[str] = None
+
+
+class LinkItem(BaseModel):
+    url: str
+    text: str = ""
+    rel: str = ""
+    title: str = ""
+
+
+class ImageItem(BaseModel):
+    url: str
+    alt: str = ""
+    title: str = ""
+    width: str = ""
+    height: str = ""
+
+
+class LinksData(BaseModel):
+    internal: list[LinkItem] = Field(default_factory=list)
+    external: list[LinkItem] = Field(default_factory=list)
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def total_count(self) -> int:
+        return len(self.internal) + len(self.external)
+
+
+class ImagesData(BaseModel):
+    images: list[ImageItem] = Field(default_factory=list)
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def total_count(self) -> int:
+        return len(self.images)
+
+
+class PerformanceMetrics(BaseModel):
+    total_time: float
+    download_time: float
+    parsing_time: float
+    extraction_time: float
+    content_size: int
+    cache_hit: bool = False
+    scrapy_used: bool = False
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def throughput_kbps(self) -> float:
+        if self.download_time > 0:
+            return round(self.content_size / 1024 / self.download_time, 2)
+        return 0.0
+
+
+class ScrapeResponse(BaseModel):
+    success: bool
+    worker_id: str
+    url: str
+    final_url: Optional[str] = None
+    status_code: Optional[int] = None
+    method_used: Optional[ScrapingMethod] = None
+    content: Optional[ContentData] = None
+    metadata: Optional[MetadataData] = None
+    links: Optional[LinksData] = None
+    images: Optional[ImagesData] = None
+    performance: PerformanceMetrics
+    error: Optional[str] = None
+    timestamp: str = Field(
+        default_factory=lambda: datetime.now(timezone.utc).isoformat()
+    )
+
+    @model_validator(mode="after")
+    def error_requires_failure(self) -> "ScrapeResponse":
+        if self.error and self.success:
+            raise ValueError("Un ScrapeResponse avec error doit avoir success=False")
+        return self
+
+
+class HealthResponse(BaseModel):
+    status: str = "healthy"
+    worker_id: str
+    uptime_seconds: float
+    total_requests: int
+    active_requests: int
+    cache_size: int
+    cache_hit_rate: float
+    avg_response_time: float
+    error_rate: float
+    methods_available: list[ScrapingMethod] = Field(
+        default_factory=lambda: list(ScrapingMethod)
+    )
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def is_degraded(self) -> bool:
+        return self.error_rate > 0.3 or self.status != "healthy"
+
+
+# Singleton settings
+settings = Settings()

requirements-4.txt

@@ -0,0 +1,57 @@
+# Framework web
+fastapi==0.115.0
+uvicorn[standard]==0.30.6
+uvloop==0.20.0
+httptools==0.6.4
+
+# Validation
+pydantic==2.9.2
+pydantic-settings==2.5.2
+orjson==3.10.7
+
+# === SCRAPY (moteur de crawl principal) ===
+scrapy==2.11.2
+twisted==24.7.0          # reactor asyncio intégré dans Scrapy 2.11+
+pyopenssl==24.2.1        # HTTPS via Twisted
+service-identity==24.2.0 # validation des certificats
+
+# Scraping fallback
+curl-cffi==0.7.3
+httpx[http2]==0.27.2
+cloudscraper==1.2.71
+aiohttp==3.10.5
+
+# Parsing
+selectolax==0.3.21
+lxml==5.3.0
+beautifulsoup4==4.12.3
+cssselect==1.2.0
+
+# Extraction de contenu
+trafilatura==1.12.0
+readability-lxml==0.8.1
+justext==3.0.1
+
+# Nettoyage / détection
+charset-normalizer==3.4.0
+ftfy==6.2.3
+langdetect==1.0.9
+w3lib==2.2.1
+
+# Async
+aiofiles==24.1.0
+
+# Monitoring
+prometheus-client==0.21.0
+structlog==24.4.0
+
+# Retry
+tenacity==9.0.0
+
+# Compression
+brotli==1.1.0
+zstandard==0.23.0
+
+# Utilitaires
+python-dotenv==1.0.1
+python-multipart==0.0.12

scraper-3.py

@@ -0,0 +1,268 @@
+"""
+Moteur Scrapy - spider, middlewares, pipelines, runner asyncio-compatible.
+
+L'astuce clé : Scrapy tourne sur Twisted, mais on l'intègre dans FastAPI
+(asyncio) via scrapy.utils.reactor.install_reactor('twisted.internet.asyncioreactor')
+appelé AVANT tout import de Scrapy. Le runner est lancé dans un thread dédié
+pour ne pas bloquer la boucle asyncio principale.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import time
+from concurrent.futures import ThreadPoolExecutor
+from typing import Any, Optional
+from urllib.parse import urlparse
+
+# Doit être le tout premier import Scrapy pour choisir l'asyncio reactor
+import scrapy.utils.reactor  # noqa: F401  (side-effect import order matters)
+from scrapy import Spider, signals
+from scrapy.crawler import CrawlerRunner
+from scrapy.http import Response
+from scrapy.item import Field, Item
+from scrapy.utils.project import get_project_settings
+from twisted.internet import defer
+
+from models import settings
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Scrapy Items
+# ---------------------------------------------------------------------------
+
+
+class PageItem(Item):
+    """Item Scrapy transportant toutes les données d'une page scrappée."""
+
+    url = Field()
+    final_url = Field()
+    status_code = Field()
+    html = Field()
+    download_time = Field()
+    error = Field()
+
+
+# ---------------------------------------------------------------------------
+# Scrapy Settings factory
+# ---------------------------------------------------------------------------
+
+
+def make_scrapy_settings(request_timeout: int, verify_ssl: bool) -> dict[str, Any]:
+    """Construit les settings Scrapy depuis notre config Pydantic."""
+    return {
+        # Identification
+        "BOT_NAME": "sota-scraper",
+        "USER_AGENT": settings.user_agent,
+        # Concurrence et politesse
+        "CONCURRENT_REQUESTS": settings.scrapy_concurrent_requests,
+        "CONCURRENT_REQUESTS_PER_DOMAIN": settings.scrapy_concurrent_per_domain,
+        "DOWNLOAD_DELAY": settings.scrapy_download_delay,
+        "RANDOMIZE_DOWNLOAD_DELAY": True,  # ±50% du DOWNLOAD_DELAY
+        # Timeouts
+        "DOWNLOAD_TIMEOUT": request_timeout,
+        # Redirections
+        "REDIRECT_ENABLED": settings.follow_redirects,
+        "REDIRECT_MAX_TIMES": 10,
+        # SSL
+        "VERIFY_SSL": verify_ssl,
+        # Retry middleware intégré
+        "RETRY_ENABLED": True,
+        "RETRY_TIMES": settings.max_retries,
+        "RETRY_HTTP_CODES": [429, 500, 502, 503, 504, 522, 524, 408],
+        "RETRY_BACKOFF_BASE": settings.retry_backoff,
+        # Compression automatique
+        "COMPRESSION_ENABLED": True,
+        # Désactiver robots.txt (scraper général)
+        "ROBOTSTXT_OBEY": False,
+        # Cookies désactivés par défaut (moins de fingerprinting)
+        "COOKIES_ENABLED": False,
+        # Logging minimal (structlog gère le reste)
+        "LOG_ENABLED": False,
+        # Telnet console désactivée
+        "TELNETCONSOLE_ENABLED": False,
+        # Reactor asyncio
+        "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+        # Middlewares downloader (ordre : 100 = premier)
+        "DOWNLOADER_MIDDLEWARES": {
+            "scrapy.downloadermiddlewares.useragent.UserAgentMiddleware": None,
+            "scrapy.downloadermiddlewares.retry.RetryMiddleware": 550,
+            "scrapy.downloadermiddlewares.redirect.RedirectMiddleware": 600,
+            "scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware": 810,
+        },
+        # Pas de pipeline ici — on récupère l'item via signal dans le runner
+        "ITEM_PIPELINES": {},
+    }
+
+
+# ---------------------------------------------------------------------------
+# Spider générique
+# ---------------------------------------------------------------------------
+
+
+class SinglePageSpider(Spider):
+    """
+    Spider minimaliste : une URL → un PageItem.
+
+    Utilisé par ScrapyRunner pour chaque requête individuelle.
+    On lui passe les headers custom via `custom_headers`.
+    """
+
+    name = "single_page"
+    # custom_settings est surchargé à l'instanciation via kwargs
+
+    def __init__(
+        self,
+        url: str,
+        custom_headers: Optional[dict[str, str]] = None,
+        *args: Any,
+        **kwargs: Any,
+    ):
+        super().__init__(*args, **kwargs)
+        self.start_urls = [url]
+        self.custom_headers = custom_headers or {}
+        self._download_start: float = 0.0
+
+    def start_requests(self):
+        headers = {
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+            "Accept-Language": "en-US,en;q=0.9,fr;q=0.8",
+            "Accept-Encoding": "gzip, deflate, br",
+            "DNT": "1",
+            "Connection": "keep-alive",
+            "Upgrade-Insecure-Requests": "1",
+            **self.custom_headers,
+        }
+        self._download_start = time.perf_counter()
+        yield scrapy.Request(
+            url=self.start_urls[0],
+            headers=headers,
+            callback=self.parse,
+            errback=self.errback,
+            dont_filter=True,
+        )
+
+    def parse(self, response: Response):
+        download_time = time.perf_counter() - self._download_start
+        yield PageItem(
+            url=self.start_urls[0],
+            final_url=str(response.url),
+            status_code=response.status,
+            html=response.text,
+            download_time=round(download_time, 4),
+            error=None,
+        )
+
+    def errback(self, failure):
+        download_time = time.perf_counter() - self._download_start
+        logger.warning("scrapy_error url=%s err=%s", self.start_urls[0], repr(failure))
+        yield PageItem(
+            url=self.start_urls[0],
+            final_url=self.start_urls[0],
+            status_code=None,
+            html="",
+            download_time=round(download_time, 4),
+            error=str(failure.value),
+        )
+
+
+# ---------------------------------------------------------------------------
+# Runner asyncio-compatible
+# ---------------------------------------------------------------------------
+
+
+class ScrapyRunner:
+    """
+    Exécute un crawl Scrapy et retourne le premier PageItem via un Future asyncio.
+
+    Architecture :
+    - Scrapy / Twisted tourne dans un ThreadPoolExecutor dédié (1 thread).
+    - La boucle asyncio principale attend via asyncio.Future.
+    - Les résultats sont transmis via call_soon_threadsafe pour rester thread-safe.
+    """
+
+    def __init__(self) -> None:
+        self._executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="scrapy")
+        self._loop: Optional[asyncio.AbstractEventLoop] = None
+
+    def _get_loop(self) -> asyncio.AbstractEventLoop:
+        if self._loop is None:
+            self._loop = asyncio.get_event_loop()
+        return self._loop
+
+    async def fetch(
+        self,
+        url: str,
+        *,
+        timeout: int = 30,
+        verify_ssl: bool = True,
+        custom_headers: Optional[dict[str, str]] = None,
+    ) -> PageItem:
+        """
+        Lance un crawl Scrapy pour `url` et retourne le PageItem résultant.
+        Lève une exception si le crawl échoue complètement.
+        """
+        loop = asyncio.get_running_loop()
+        future: asyncio.Future[PageItem] = loop.create_future()
+
+        def _run_in_thread() -> None:
+            """Exécution bloquante dans le thread Scrapy/Twisted."""
+            try:
+                from twisted.internet import reactor  # type: ignore
+
+                scrapy_cfg = get_project_settings()
+                scrapy_cfg.update(make_scrapy_settings(timeout, verify_ssl))
+
+                runner = CrawlerRunner(scrapy_cfg)
+                collected: list[PageItem] = []
+
+                crawler = runner.create_crawler(SinglePageSpider)
+
+                def _on_item(item: PageItem, response: Any, spider: Any) -> None:
+                    collected.append(item)
+
+                def _on_finished(_: Any) -> None:
+                    result = collected[0] if collected else PageItem(
+                        url=url,
+                        final_url=url,
+                        status_code=None,
+                        html="",
+                        download_time=0.0,
+                        error="Aucun item collecté par Scrapy",
+                    )
+                    loop.call_soon_threadsafe(
+                        future.set_result, result  # type: ignore[arg-type]
+                    )
+
+                def _on_error(failure: Any) -> None:
+                    exc = failure.value if hasattr(failure, "value") else Exception(str(failure))
+                    loop.call_soon_threadsafe(future.set_exception, exc)
+
+                crawler.signals.connect(_on_item, signal=signals.item_scraped)
+
+                d: defer.Deferred = runner.crawl(
+                    crawler,
+                    url=url,
+                    custom_headers=custom_headers or {},
+                )
+                d.addCallback(_on_finished)
+                d.addErrback(_on_error)
+
+                # Démarrer le reactor si pas déjà démarré
+                if not reactor.running:  # type: ignore[attr-defined]
+                    reactor.run(installSignalHandlers=False)  # type: ignore[attr-defined]
+
+            except Exception as exc:
+                loop.call_soon_threadsafe(future.set_exception, exc)
+
+        await loop.run_in_executor(self._executor, _run_in_thread)
+        return await future
+
+    async def shutdown(self) -> None:
+        self._executor.shutdown(wait=False)
+
+
+# Singleton partagé par l'application
+scrapy_runner = ScrapyRunner()

utils-4.py

@@ -0,0 +1,395 @@
+"""
+Pipeline d'extraction de contenu — version refactorisée.
+
+- TextCleaner     : nettoyage profond du texte (inchangé, déjà très bon)
+- ContentCleaner  : extraction multi-méthodes (trafilatura → readability → justext)
+- URLValidator    : validation et normalisation d'URL
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+import re
+from typing import Any, Optional
+from urllib.parse import parse_qs, urlencode, urljoin, urlparse, urlunparse
+
+import ftfy
+import justext
+from langdetect import detect_langs
+from readability import Document
+from selectolax.parser import HTMLParser
+from trafilatura import bare_extraction
+from w3lib.html import remove_tags, replace_entities
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# TextCleaner
+# ---------------------------------------------------------------------------
+
+
+class TextCleaner:
+    """Pipeline de nettoyage de texte ultra-agressif."""
+
+    MD_IMAGE = re.compile(r"!\[.*?\]\(.*?\)", re.DOTALL)
+    MD_LINK = re.compile(r"\[([^\]]*)\]\([^)]*\)", re.DOTALL)
+    WIKI_REF = re.compile(r"\[\[?\d+\]?\]\([^)]*\)|\[\[?\d+\]?\]")
+    RAW_URL = re.compile(r"https?://[^\s\)\]\,\"\'<>]+|//[^\s\)\]\,\"\'<>]+")
+    HTML_TAGS = re.compile(r"<[^>]+>")
+    MULTI_SPACE = re.compile(r" {2,}")
+    MULTI_NEWLINE = re.compile(r"\n{3,}")
+    CONTROL_CHARS = re.compile(r"[\x00-\x08\x0b-\x0c\x0e-\x1f\x7f]")
+    JUNK_LINE = re.compile(r"^\s*[\|\-\=\*\#\~\^]{2,}\s*$", re.MULTILINE)
+    TABLE_ROW = re.compile(r"^\|.*\|$", re.MULTILINE)
+    ONLY_PUNCTUATION = re.compile(r"^[\d\s\.\,\;\:\!\?\-\|\=\[\]\(\)]+$")
+
+    @classmethod
+    def deep_clean(cls, text: str) -> str:
+        if not text:
+            return ""
+
+        text = ftfy.fix_text(text)
+        text = replace_entities(text)
+        text = cls.MD_IMAGE.sub("", text)
+        text = cls.WIKI_REF.sub("", text)
+        text = cls.MD_LINK.sub(r"\1", text)
+        text = cls.RAW_URL.sub("", text)
+        text = cls.HTML_TAGS.sub("", text)
+        text = cls.CONTROL_CHARS.sub("", text)
+        text = cls.TABLE_ROW.sub("", text)
+        text = cls.JUNK_LINE.sub("", text)
+
+        lines = []
+        for line in text.splitlines():
+            line = line.strip()
+            if len(line) < 2:
+                continue
+            if cls.ONLY_PUNCTUATION.match(line):
+                continue
+            lines.append(line)
+
+        text = "\n".join(lines)
+        text = cls.MULTI_SPACE.sub(" ", text)
+        text = cls.MULTI_NEWLINE.sub("\n\n", text)
+        return text.strip()
+
+    @classmethod
+    def extract_clean_sentences(cls, text: str, min_length: int = 30) -> str:
+        text = cls.deep_clean(text)
+        paragraphs = text.split("\n\n")
+        valid = [p.strip() for p in paragraphs if len(p.strip()) >= min_length]
+        return "\n\n".join(valid)
+
+
+# ---------------------------------------------------------------------------
+# ContentCleaner
+# ---------------------------------------------------------------------------
+
+
+class ContentCleaner:
+    """Extraction et nettoyage de contenu HTML."""
+
+    UNWANTED_CSS_SELECTORS: list[str] = [
+        "sup.reference",
+        "div.reflist",
+        "div.navbox",
+        "div.toc",
+        "div.hatnote",
+        "table.navbox",
+        "table.wikitable",
+        "div.mw-references-wrap",
+        "ol.references",
+        "span.mw-editsection",
+        "div.sidebar",
+        "div.noprint",
+        ".navigation-not-searchable",
+        "script",
+        "style",
+        "noscript",
+        "iframe",
+        "embed",
+        "object",
+        "svg",
+        "canvas",
+        "head",
+    ]
+
+    @classmethod
+    def clean_html_fast(cls, html: str) -> str:
+        """Supprime les éléments parasites avant extraction."""
+        try:
+            tree = HTMLParser(html)
+            for selector in cls.UNWANTED_CSS_SELECTORS:
+                try:
+                    for node in tree.css(selector):
+                        node.decompose()
+                except Exception:
+                    pass
+            return tree.html or html
+        except Exception as exc:
+            logger.warning("clean_html_fast error=%s", exc)
+            return html
+
+    @classmethod
+    def extract_main_content(cls, html: str, url: str) -> dict[str, Any]:
+        """
+        Extraction multi-méthodes : trafilatura → readability → justext → basic.
+        Retourne toujours un dict même si toutes les méthodes échouent.
+        """
+        result: dict[str, Any] = {
+            "text": "",
+            "title": "",
+            "author": "",
+            "date": None,
+            "description": "",
+            "language": "unknown",
+            "method": "unknown",
+        }
+
+        clean_html = cls.clean_html_fast(html)
+
+        # --- Méthode 1 : trafilatura (SOTA précision) ---
+        try:
+            extracted = bare_extraction(
+                clean_html,
+                url=url,
+                include_comments=False,
+                include_tables=False,
+                include_images=False,
+                include_links=False,
+                deduplicate=True,
+                favor_precision=True,
+                no_fallback=False,
+            )
+            if extracted and len(extracted.get("text") or "") > 100:
+                clean_text = TextCleaner.deep_clean(extracted["text"])
+                result.update(
+                    {
+                        "text": clean_text,
+                        "title": extracted.get("title") or "",
+                        "author": extracted.get("author") or "",
+                        "date": str(extracted["date"]) if extracted.get("date") else None,
+                        "description": extracted.get("description") or "",
+                        "method": "trafilatura",
+                    }
+                )
+                result["language"] = cls._detect_language(clean_text)
+                return result
+        except Exception as exc:
+            logger.debug("trafilatura_failed error=%s", exc)
+
+        # --- Méthode 2 : readability ---
+        try:
+            doc = Document(clean_html)
+            raw_text = remove_tags(doc.summary())
+            clean_text = TextCleaner.deep_clean(raw_text)
+            if len(clean_text) > 50:
+                result.update(
+                    {
+                        "text": clean_text,
+                        "title": doc.title(),
+                        "method": "readability",
+                    }
+                )
+                result["language"] = cls._detect_language(clean_text)
+                return result
+        except Exception as exc:
+            logger.debug("readability_failed error=%s", exc)
+
+        # --- Méthode 3 : justext ---
+        try:
+            paragraphs = justext.justext(
+                clean_html.encode("utf-8", errors="replace"),
+                justext.get_stoplist("English"),
+                length_low=50,
+                length_high=200,
+                stopwords_low=0.20,
+                stopwords_high=0.30,
+                max_link_density=0.3,
+                no_headings=False,
+            )
+            texts = [p.text for p in paragraphs if not p.is_boilerplate]
+            clean_text = TextCleaner.deep_clean("\n\n".join(texts))
+            if clean_text:
+                result.update({"text": clean_text, "method": "justext"})
+                result["language"] = cls._detect_language(clean_text)
+                return result
+        except Exception as exc:
+            logger.debug("justext_failed error=%s", exc)
+
+        # --- Fallback ultime ---
+        result["text"] = TextCleaner.deep_clean(remove_tags(clean_html))
+        result["method"] = "basic"
+        return result
+
+    @staticmethod
+    def _detect_language(text: str) -> str:
+        try:
+            if text:
+                langs = detect_langs(text[:500])
+                return langs[0].lang if langs else "unknown"
+        except Exception:
+            pass
+        return "unknown"
+
+    @staticmethod
+    def normalize_text(text: str) -> str:
+        return TextCleaner.deep_clean(text)
+
+    @staticmethod
+    def extract_metadata(html: str) -> dict[str, Any]:
+        """Open Graph + Twitter Cards + meta standards."""
+        metadata: dict[str, Any] = {}
+        try:
+            tree = HTMLParser(html)
+
+            for meta in tree.css('meta[property^="og:"]'):
+                prop = meta.attributes.get("property", "").replace("og:", "").strip()
+                content = meta.attributes.get("content", "").strip()
+                if prop and content:
+                    metadata[f"og_{prop}"] = content
+
+            for meta in tree.css('meta[name^="twitter:"]'):
+                name = meta.attributes.get("name", "").replace("twitter:", "").strip()
+                content = meta.attributes.get("content", "").strip()
+                if name and content:
+                    metadata[f"twitter_{name}"] = content
+
+            for meta in tree.css("meta[name]"):
+                name = meta.attributes.get("name", "").strip()
+                content = meta.attributes.get("content", "").strip()
+                if name in {"description", "keywords", "author"} and content:
+                    metadata[name] = content
+
+            canonical = tree.css_first('link[rel="canonical"]')
+            if canonical:
+                href = canonical.attributes.get("href", "").strip()
+                if href:
+                    metadata["canonical"] = href
+
+            title_node = tree.css_first("title")
+            if title_node and not metadata.get("og_title"):
+                metadata["page_title"] = title_node.text(strip=True)
+
+        except Exception as exc:
+            logger.warning("extract_metadata error=%s", exc)
+
+        return metadata
+
+    @staticmethod
+    def extract_links(html: str, base_url: str) -> list[dict[str, str]]:
+        links: list[dict[str, str]] = []
+        seen: set[str] = set()
+        try:
+            tree = HTMLParser(html)
+            for link in tree.css("a[href]"):
+                href = link.attributes.get("href", "").strip()
+                if not href or href.startswith(("#", "javascript:", "mailto:", "tel:")):
+                    continue
+                try:
+                    abs_url = urljoin(base_url, href)
+                except Exception:
+                    continue
+                if abs_url in seen:
+                    continue
+                seen.add(abs_url)
+                parsed = urlparse(abs_url)
+                if parsed.scheme not in {"http", "https"}:
+                    continue
+                links.append(
+                    {
+                        "url": abs_url,
+                        "text": (link.text(strip=True) or "")[:200],
+                        "rel": link.attributes.get("rel", ""),
+                        "title": link.attributes.get("title", ""),
+                    }
+                )
+        except Exception as exc:
+            logger.warning("extract_links error=%s", exc)
+        return links
+
+    @staticmethod
+    def extract_images(html: str, base_url: str) -> list[dict[str, str]]:
+        images: list[dict[str, str]] = []
+        seen: set[str] = set()
+        src_attrs = ("src", "data-src", "data-lazy-src", "data-original", "data-lazy")
+        try:
+            tree = HTMLParser(html)
+            for img in tree.css("img"):
+                src = next(
+                    (img.attributes.get(a, "") for a in src_attrs if img.attributes.get(a)),
+                    "",
+                ).strip()
+                if not src or src.startswith("data:"):
+                    continue
+                try:
+                    abs_url = urljoin(base_url, src)
+                except Exception:
+                    continue
+                if abs_url in seen:
+                    continue
+                seen.add(abs_url)
+                if urlparse(abs_url).scheme not in {"http", "https"}:
+                    continue
+                images.append(
+                    {
+                        "url": abs_url,
+                        "alt": img.attributes.get("alt", "").strip(),
+                        "title": img.attributes.get("title", "").strip(),
+                        "width": img.attributes.get("width", ""),
+                        "height": img.attributes.get("height", ""),
+                    }
+                )
+        except Exception as exc:
+            logger.warning("extract_images error=%s", exc)
+        return images
+
+    @staticmethod
+    def compute_content_hash(text: str) -> str:
+        return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
+# ---------------------------------------------------------------------------
+# URLValidator
+# ---------------------------------------------------------------------------
+
+
+class URLValidator:
+    BLOCKED_EXTENSIONS: frozenset[str] = frozenset(
+        {
+            ".pdf", ".zip", ".exe", ".dmg", ".pkg", ".deb", ".rpm",
+            ".jpg", ".jpeg", ".png", ".gif", ".svg", ".webp", ".ico",
+            ".mp4", ".avi", ".mov", ".mp3", ".wav", ".flac",
+            ".css", ".js", ".woff", ".woff2", ".ttf", ".eot",
+        }
+    )
+    ALLOWED_SCHEMES: frozenset[str] = frozenset({"http", "https"})
+
+    @classmethod
+    def is_valid_url(cls, url: str) -> bool:
+        try:
+            parsed = urlparse(url)
+            if parsed.scheme not in cls.ALLOWED_SCHEMES:
+                return False
+            if not parsed.netloc:
+                return False
+            path_lower = parsed.path.lower()
+            if any(path_lower.endswith(ext) for ext in cls.BLOCKED_EXTENSIONS):
+                return False
+            return True
+        except Exception:
+            return False
+
+    @staticmethod
+    def normalize_url(url: str) -> str:
+        try:
+            parsed = urlparse(url)._replace(fragment="")
+            if parsed.query:
+                params = parse_qs(parsed.query)
+                new_query = urlencode(sorted(params.items()), doseq=True)
+                parsed = parsed._replace(query=new_query)
+            return urlunparse(parsed)
+        except Exception:
+            return url