""" Admin routes - Scraper management, system health, source CRUD. """ import uuid from datetime import datetime from fastapi import APIRouter, HTTPException, Query, status from pydantic import BaseModel from sqlalchemy import func, select from app.api.deps import DBSession, SuperUser from app.models.job import Job, JobStatus from app.models.scraper import JobSource, ScrapeRun, ScrapeRunStatus, SourceStatus from app.models.user import User from app.worker.tasks import scrape_single_source router = APIRouter(prefix="/admin", tags=["admin"]) # ----- Schemas ----- class SourceCreate(BaseModel): name: str source_type: str # ats_api, careers_page, sitemap, rss domain: str base_url: str ats_vendor: str | None = None ats_config: dict | None = None crawl_frequency_hours: int = 24 max_pages_per_run: int = 100 request_delay_seconds: float = 2.0 requires_browser: bool = False compliance_notes: str | None = None class SourceUpdate(BaseModel): name: str | None = None status: str | None = None crawl_frequency_hours: int | None = None max_pages_per_run: int | None = None request_delay_seconds: float | None = None ats_config: dict | None = None compliance_notes: str | None = None class SourceResponse(BaseModel): id: str name: str source_type: str domain: str base_url: str ats_vendor: str | None = None status: str crawl_frequency_hours: int last_crawled_at: datetime | None = None last_success_at: datetime | None = None consecutive_failures: int total_jobs_found: int robots_allows_crawl: bool | None = None created_at: datetime class Config: from_attributes = True class ScrapeRunResponse(BaseModel): id: str source_id: str status: str started_at: datetime | None = None completed_at: datetime | None = None pages_fetched: int jobs_found: int jobs_new: int jobs_updated: int error_message: str | None = None created_at: datetime class Config: from_attributes = True class SystemHealthResponse(BaseModel): total_jobs: int active_jobs: int expired_jobs: int total_sources: int active_sources: int total_users: int total_scrape_runs: int recent_failures: int last_scrape_at: datetime | None = None # ----- Routes ----- @router.get("/health", response_model=SystemHealthResponse) async def system_health(db: DBSession, user: SuperUser): """Get system health overview.""" total_jobs = await db.scalar(select(func.count(Job.id))) active_jobs = await db.scalar(select(func.count(Job.id)).where(Job.status == JobStatus.ACTIVE)) expired_jobs = await db.scalar(select(func.count(Job.id)).where(Job.status == JobStatus.EXPIRED)) total_sources = await db.scalar(select(func.count(JobSource.id))) active_sources = await db.scalar( select(func.count(JobSource.id)).where(JobSource.status == SourceStatus.ACTIVE) ) total_users = await db.scalar(select(func.count(User.id))) total_runs = await db.scalar(select(func.count(ScrapeRun.id))) # Recent failures (last 24 hours) from datetime import timedelta cutoff = datetime.utcnow() - timedelta(hours=24) recent_failures = await db.scalar( select(func.count(ScrapeRun.id)).where( ScrapeRun.status == ScrapeRunStatus.FAILED, ScrapeRun.created_at >= cutoff, ) ) # Last scrape last_run = await db.execute( select(ScrapeRun).order_by(ScrapeRun.completed_at.desc().nullslast()).limit(1) ) last = last_run.scalar_one_or_none() return SystemHealthResponse( total_jobs=total_jobs or 0, active_jobs=active_jobs or 0, expired_jobs=expired_jobs or 0, total_sources=total_sources or 0, active_sources=active_sources or 0, total_users=total_users or 0, total_scrape_runs=total_runs or 0, recent_failures=recent_failures or 0, last_scrape_at=last.completed_at if last else None, ) @router.get("/sources", response_model=list[SourceResponse]) async def list_sources(db: DBSession, user: SuperUser): """List all job sources.""" result = await db.execute(select(JobSource).order_by(JobSource.name)) sources = result.scalars().all() return [ SourceResponse( id=str(s.id), name=s.name, source_type=s.source_type, domain=s.domain, base_url=s.base_url, ats_vendor=s.ats_vendor, status=s.status, crawl_frequency_hours=s.crawl_frequency_hours, last_crawled_at=s.last_crawled_at, last_success_at=s.last_success_at, consecutive_failures=s.consecutive_failures, total_jobs_found=s.total_jobs_found, robots_allows_crawl=s.robots_allows_crawl, created_at=s.created_at, ) for s in sources ] @router.post("/sources", response_model=SourceResponse, status_code=status.HTTP_201_CREATED) async def create_source(data: SourceCreate, db: DBSession, user: SuperUser): """Create a new job source.""" source = JobSource( name=data.name, source_type=data.source_type, domain=data.domain, base_url=data.base_url, ats_vendor=data.ats_vendor, ats_config=data.ats_config, crawl_frequency_hours=data.crawl_frequency_hours, max_pages_per_run=data.max_pages_per_run, request_delay_seconds=data.request_delay_seconds, requires_browser=data.requires_browser, compliance_notes=data.compliance_notes, ) db.add(source) await db.flush() await db.refresh(source) return SourceResponse( id=str(source.id), name=source.name, source_type=source.source_type, domain=source.domain, base_url=source.base_url, ats_vendor=source.ats_vendor, status=source.status, crawl_frequency_hours=source.crawl_frequency_hours, last_crawled_at=source.last_crawled_at, last_success_at=source.last_success_at, consecutive_failures=source.consecutive_failures, total_jobs_found=source.total_jobs_found, robots_allows_crawl=source.robots_allows_crawl, created_at=source.created_at, ) @router.patch("/sources/{source_id}", response_model=SourceResponse) async def update_source(source_id: uuid.UUID, data: SourceUpdate, db: DBSession, user: SuperUser): """Update a job source.""" result = await db.execute(select(JobSource).where(JobSource.id == source_id)) source = result.scalar_one_or_none() if not source: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Source not found") update_data = data.model_dump(exclude_unset=True) for field, value in update_data.items(): setattr(source, field, value) await db.flush() await db.refresh(source) return SourceResponse( id=str(source.id), name=source.name, source_type=source.source_type, domain=source.domain, base_url=source.base_url, ats_vendor=source.ats_vendor, status=source.status, crawl_frequency_hours=source.crawl_frequency_hours, last_crawled_at=source.last_crawled_at, last_success_at=source.last_success_at, consecutive_failures=source.consecutive_failures, total_jobs_found=source.total_jobs_found, robots_allows_crawl=source.robots_allows_crawl, created_at=source.created_at, ) @router.post("/sources/{source_id}/scrape") async def trigger_scrape(source_id: uuid.UUID, db: DBSession, user: SuperUser): """Manually trigger a scrape for a source.""" result = await db.execute(select(JobSource).where(JobSource.id == source_id)) source = result.scalar_one_or_none() if not source: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Source not found") # Queue the scrape task scrape_single_source.delay(str(source_id)) return {"detail": f"Scrape queued for {source.name}", "source_id": str(source_id)} @router.get("/sources/{source_id}/runs", response_model=list[ScrapeRunResponse]) async def list_source_runs( source_id: uuid.UUID, db: DBSession, user: SuperUser, limit: int = Query(20, le=100) ): """List recent scrape runs for a source.""" result = await db.execute( select(ScrapeRun) .where(ScrapeRun.source_id == source_id) .order_by(ScrapeRun.created_at.desc()) .limit(limit) ) runs = result.scalars().all() return [ ScrapeRunResponse( id=str(r.id), source_id=str(r.source_id), status=r.status, started_at=r.started_at, completed_at=r.completed_at, pages_fetched=r.pages_fetched, jobs_found=r.jobs_found, jobs_new=r.jobs_new, jobs_updated=r.jobs_updated, error_message=r.error_message, created_at=r.created_at, ) for r in runs ] @router.delete("/sources/{source_id}", status_code=status.HTTP_204_NO_CONTENT) async def delete_source(source_id: uuid.UUID, db: DBSession, user: SuperUser): """Delete a job source.""" result = await db.execute(select(JobSource).where(JobSource.id == source_id)) source = result.scalar_one_or_none() if not source: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Source not found") await db.delete(source)