backend/app/models/scraper.py

"""
Scraper models - track scrape runs, sources, and compliance.
"""
import uuid
from datetime import datetime
from enum import Enum

from sqlalchemy import Boolean, DateTime, Float, Integer, String, Text, func
from sqlalchemy.dialects.postgresql import JSONB, UUID
from sqlalchemy.orm import Mapped, mapped_column

from app.core.database import Base


class SourceStatus(str, Enum):
    ACTIVE = "active"
    PAUSED = "paused"
    BLOCKED = "blocked"
    DEPRECATED = "deprecated"


class ScrapeRunStatus(str, Enum):
    PENDING = "pending"
    RUNNING = "running"
    COMPLETED = "completed"
    FAILED = "failed"
    CANCELLED = "cancelled"


class JobSource(Base):
    """Configuration for each job source/ATS endpoint."""
    __tablename__ = "job_sources"

    id: Mapped[uuid.UUID] = mapped_column(
        UUID(as_uuid=True), primary_key=True, default=uuid.uuid4
    )
    name: Mapped[str] = mapped_column(String(256), nullable=False)
    source_type: Mapped[str] = mapped_column(String(50), nullable=False)  # ats_api, careers_page, sitemap, rss, manual
    domain: Mapped[str] = mapped_column(String(256), nullable=False, index=True)
    base_url: Mapped[str] = mapped_column(String(2048), nullable=False)

    # ATS specifics
    ats_vendor: Mapped[str | None] = mapped_column(String(50), nullable=True)  # greenhouse, lever, workday, etc.
    ats_config: Mapped[dict | None] = mapped_column(JSONB, nullable=True)  # API endpoints, params, auth

    # Compliance
    robots_txt_url: Mapped[str | None] = mapped_column(String(512), nullable=True)
    robots_allows_crawl: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
    robots_crawl_delay: Mapped[float | None] = mapped_column(Float, nullable=True)
    terms_url: Mapped[str | None] = mapped_column(String(512), nullable=True)
    terms_reviewed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
    terms_allows_crawl: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
    compliance_notes: Mapped[str | None] = mapped_column(Text, nullable=True)

    # Crawl config
    crawl_frequency_hours: Mapped[int] = mapped_column(Integer, default=24)
    max_pages_per_run: Mapped[int] = mapped_column(Integer, default=100)
    request_delay_seconds: Mapped[float] = mapped_column(Float, default=2.0)
    requires_browser: Mapped[bool] = mapped_column(Boolean, default=False)
    custom_headers: Mapped[dict | None] = mapped_column(JSONB, nullable=True)

    # Status
    status: Mapped[str] = mapped_column(String(20), default=SourceStatus.ACTIVE)
    last_crawled_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
    last_success_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
    consecutive_failures: Mapped[int] = mapped_column(Integer, default=0)
    total_jobs_found: Mapped[int] = mapped_column(Integer, default=0)

    created_at: Mapped[datetime] = mapped_column(
        DateTime(timezone=True), server_default=func.now()
    )
    updated_at: Mapped[datetime] = mapped_column(
        DateTime(timezone=True), server_default=func.now(), onupdate=func.now()
    )


class ScrapeRun(Base):
    """Individual scrape execution log."""
    __tablename__ = "scrape_runs"

    id: Mapped[uuid.UUID] = mapped_column(
        UUID(as_uuid=True), primary_key=True, default=uuid.uuid4
    )
    source_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), nullable=False, index=True)

    status: Mapped[str] = mapped_column(String(20), default=ScrapeRunStatus.PENDING)
    started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
    completed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)

    # Metrics
    pages_fetched: Mapped[int] = mapped_column(Integer, default=0)
    pages_failed: Mapped[int] = mapped_column(Integer, default=0)
    jobs_found: Mapped[int] = mapped_column(Integer, default=0)
    jobs_new: Mapped[int] = mapped_column(Integer, default=0)
    jobs_updated: Mapped[int] = mapped_column(Integer, default=0)
    jobs_expired: Mapped[int] = mapped_column(Integer, default=0)
    duplicates_skipped: Mapped[int] = mapped_column(Integer, default=0)

    # Error tracking
    error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
    error_details: Mapped[dict | None] = mapped_column(JSONB, nullable=True)

    # Compliance log
    robots_status: Mapped[str | None] = mapped_column(String(20), nullable=True)  # allowed, denied, unavailable
    crawl_delay_used: Mapped[float | None] = mapped_column(Float, nullable=True)

    # R2 log reference
    log_file_key: Mapped[str | None] = mapped_column(String(512), nullable=True)

    created_at: Mapped[datetime] = mapped_column(
        DateTime(timezone=True), server_default=func.now()
    )