backend/app/api/routes/admin.py

"""
Admin routes - Scraper management, system health, source CRUD.
"""
import uuid
from datetime import datetime

from fastapi import APIRouter, HTTPException, Query, status
from pydantic import BaseModel
from sqlalchemy import func, select

from app.api.deps import DBSession, SuperUser
from app.models.job import Job, JobStatus
from app.models.scraper import JobSource, ScrapeRun, ScrapeRunStatus, SourceStatus
from app.models.user import User
from app.worker.tasks import scrape_single_source

router = APIRouter(prefix="/admin", tags=["admin"])


# ----- Schemas -----

class SourceCreate(BaseModel):
    name: str
    source_type: str  # ats_api, careers_page, sitemap, rss
    domain: str
    base_url: str
    ats_vendor: str | None = None
    ats_config: dict | None = None
    crawl_frequency_hours: int = 24
    max_pages_per_run: int = 100
    request_delay_seconds: float = 2.0
    requires_browser: bool = False
    compliance_notes: str | None = None


class SourceUpdate(BaseModel):
    name: str | None = None
    status: str | None = None
    crawl_frequency_hours: int | None = None
    max_pages_per_run: int | None = None
    request_delay_seconds: float | None = None
    ats_config: dict | None = None
    compliance_notes: str | None = None


class SourceResponse(BaseModel):
    id: str
    name: str
    source_type: str
    domain: str
    base_url: str
    ats_vendor: str | None = None
    status: str
    crawl_frequency_hours: int
    last_crawled_at: datetime | None = None
    last_success_at: datetime | None = None
    consecutive_failures: int
    total_jobs_found: int
    robots_allows_crawl: bool | None = None
    created_at: datetime

    class Config:
        from_attributes = True


class ScrapeRunResponse(BaseModel):
    id: str
    source_id: str
    status: str
    started_at: datetime | None = None
    completed_at: datetime | None = None
    pages_fetched: int
    jobs_found: int
    jobs_new: int
    jobs_updated: int
    error_message: str | None = None
    created_at: datetime

    class Config:
        from_attributes = True


class SystemHealthResponse(BaseModel):
    total_jobs: int
    active_jobs: int
    expired_jobs: int
    total_sources: int
    active_sources: int
    total_users: int
    total_scrape_runs: int
    recent_failures: int
    last_scrape_at: datetime | None = None


# ----- Routes -----

@router.get("/health", response_model=SystemHealthResponse)
async def system_health(db: DBSession, user: SuperUser):
    """Get system health overview."""
    total_jobs = await db.scalar(select(func.count(Job.id)))
    active_jobs = await db.scalar(select(func.count(Job.id)).where(Job.status == JobStatus.ACTIVE))
    expired_jobs = await db.scalar(select(func.count(Job.id)).where(Job.status == JobStatus.EXPIRED))
    total_sources = await db.scalar(select(func.count(JobSource.id)))
    active_sources = await db.scalar(
        select(func.count(JobSource.id)).where(JobSource.status == SourceStatus.ACTIVE)
    )
    total_users = await db.scalar(select(func.count(User.id)))
    total_runs = await db.scalar(select(func.count(ScrapeRun.id)))

    # Recent failures (last 24 hours)
    from datetime import timedelta
    cutoff = datetime.utcnow() - timedelta(hours=24)
    recent_failures = await db.scalar(
        select(func.count(ScrapeRun.id)).where(
            ScrapeRun.status == ScrapeRunStatus.FAILED,
            ScrapeRun.created_at >= cutoff,
        )
    )

    # Last scrape
    last_run = await db.execute(
        select(ScrapeRun).order_by(ScrapeRun.completed_at.desc().nullslast()).limit(1)
    )
    last = last_run.scalar_one_or_none()

    return SystemHealthResponse(
        total_jobs=total_jobs or 0,
        active_jobs=active_jobs or 0,
        expired_jobs=expired_jobs or 0,
        total_sources=total_sources or 0,
        active_sources=active_sources or 0,
        total_users=total_users or 0,
        total_scrape_runs=total_runs or 0,
        recent_failures=recent_failures or 0,
        last_scrape_at=last.completed_at if last else None,
    )


@router.get("/sources", response_model=list[SourceResponse])
async def list_sources(db: DBSession, user: SuperUser):
    """List all job sources."""
    result = await db.execute(select(JobSource).order_by(JobSource.name))
    sources = result.scalars().all()
    return [
        SourceResponse(
            id=str(s.id),
            name=s.name,
            source_type=s.source_type,
            domain=s.domain,
            base_url=s.base_url,
            ats_vendor=s.ats_vendor,
            status=s.status,
            crawl_frequency_hours=s.crawl_frequency_hours,
            last_crawled_at=s.last_crawled_at,
            last_success_at=s.last_success_at,
            consecutive_failures=s.consecutive_failures,
            total_jobs_found=s.total_jobs_found,
            robots_allows_crawl=s.robots_allows_crawl,
            created_at=s.created_at,
        )
        for s in sources
    ]


@router.post("/sources", response_model=SourceResponse, status_code=status.HTTP_201_CREATED)
async def create_source(data: SourceCreate, db: DBSession, user: SuperUser):
    """Create a new job source."""
    source = JobSource(
        name=data.name,
        source_type=data.source_type,
        domain=data.domain,
        base_url=data.base_url,
        ats_vendor=data.ats_vendor,
        ats_config=data.ats_config,
        crawl_frequency_hours=data.crawl_frequency_hours,
        max_pages_per_run=data.max_pages_per_run,
        request_delay_seconds=data.request_delay_seconds,
        requires_browser=data.requires_browser,
        compliance_notes=data.compliance_notes,
    )
    db.add(source)
    await db.flush()
    await db.refresh(source)

    return SourceResponse(
        id=str(source.id),
        name=source.name,
        source_type=source.source_type,
        domain=source.domain,
        base_url=source.base_url,
        ats_vendor=source.ats_vendor,
        status=source.status,
        crawl_frequency_hours=source.crawl_frequency_hours,
        last_crawled_at=source.last_crawled_at,
        last_success_at=source.last_success_at,
        consecutive_failures=source.consecutive_failures,
        total_jobs_found=source.total_jobs_found,
        robots_allows_crawl=source.robots_allows_crawl,
        created_at=source.created_at,
    )


@router.patch("/sources/{source_id}", response_model=SourceResponse)
async def update_source(source_id: uuid.UUID, data: SourceUpdate, db: DBSession, user: SuperUser):
    """Update a job source."""
    result = await db.execute(select(JobSource).where(JobSource.id == source_id))
    source = result.scalar_one_or_none()
    if not source:
        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Source not found")

    update_data = data.model_dump(exclude_unset=True)
    for field, value in update_data.items():
        setattr(source, field, value)

    await db.flush()
    await db.refresh(source)

    return SourceResponse(
        id=str(source.id),
        name=source.name,
        source_type=source.source_type,
        domain=source.domain,
        base_url=source.base_url,
        ats_vendor=source.ats_vendor,
        status=source.status,
        crawl_frequency_hours=source.crawl_frequency_hours,
        last_crawled_at=source.last_crawled_at,
        last_success_at=source.last_success_at,
        consecutive_failures=source.consecutive_failures,
        total_jobs_found=source.total_jobs_found,
        robots_allows_crawl=source.robots_allows_crawl,
        created_at=source.created_at,
    )


@router.post("/sources/{source_id}/scrape")
async def trigger_scrape(source_id: uuid.UUID, db: DBSession, user: SuperUser):
    """Manually trigger a scrape for a source."""
    result = await db.execute(select(JobSource).where(JobSource.id == source_id))
    source = result.scalar_one_or_none()
    if not source:
        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Source not found")

    # Queue the scrape task
    scrape_single_source.delay(str(source_id))
    return {"detail": f"Scrape queued for {source.name}", "source_id": str(source_id)}


@router.get("/sources/{source_id}/runs", response_model=list[ScrapeRunResponse])
async def list_source_runs(
    source_id: uuid.UUID, db: DBSession, user: SuperUser, limit: int = Query(20, le=100)
):
    """List recent scrape runs for a source."""
    result = await db.execute(
        select(ScrapeRun)
        .where(ScrapeRun.source_id == source_id)
        .order_by(ScrapeRun.created_at.desc())
        .limit(limit)
    )
    runs = result.scalars().all()
    return [
        ScrapeRunResponse(
            id=str(r.id),
            source_id=str(r.source_id),
            status=r.status,
            started_at=r.started_at,
            completed_at=r.completed_at,
            pages_fetched=r.pages_fetched,
            jobs_found=r.jobs_found,
            jobs_new=r.jobs_new,
            jobs_updated=r.jobs_updated,
            error_message=r.error_message,
            created_at=r.created_at,
        )
        for r in runs
    ]


@router.delete("/sources/{source_id}", status_code=status.HTTP_204_NO_CONTENT)
async def delete_source(source_id: uuid.UUID, db: DBSession, user: SuperUser):
    """Delete a job source."""
    result = await db.execute(select(JobSource).where(JobSource.id == source_id))
    source = result.scalar_one_or_none()
    if not source:
        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Source not found")
    await db.delete(source)