Delete app.py

app.py

@@ -1,518 +0,0 @@
-"""
-FastAPI app for the FIPI scraper service.
-"""
-
-from __future__ import annotations
-
-from datetime import datetime
-import logging
-import os
-from pathlib import Path
-import re
-import ssl
-from typing import Dict, List, Optional
-from urllib.parse import parse_qs, urlparse
-
-from bs4 import BeautifulSoup
-from dotenv import load_dotenv
-from fastapi import BackgroundTasks, FastAPI, HTTPException
-from fastapi.middleware.cors import CORSMiddleware
-from fastapi.responses import JSONResponse
-import httpx
-
-from models import (
-    CheckAnswerRequest,
-    CheckAnswerResponse,
-    ErrorResponse,
-    HealthResponse,
-    ScrapeRequest,
-    ScrapeResponse,
-    StatsResponse,
-    TaskResponse,
-)
-from scraper import FIPIScraper
-
-BASE_DIR = Path(__file__).resolve().parent
-load_dotenv(BASE_DIR / ".env")
-
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger(__name__)
-
-SUPABASE_AVAILABLE = False
-SupabaseClient = None
-
-try:
-    from supabase_client import SupabaseClient
-
-    SUPABASE_AVAILABLE = True
-except ImportError as exc:
-    logger.warning("Supabase client import failed: %s", exc)
-except Exception as exc:  # pragma: no cover - startup guard
-    logger.warning("Supabase client init failed: %s", exc)
-
-
-app = FastAPI(
-    title="AI Scraper FIPI",
-    description="Collects, stores and validates FIPI tasks.",
-    version="1.1.1-proof-20260317",
-)
-
-app.add_middleware(
-    CORSMiddleware,
-    allow_origins=["*"],
-    allow_credentials=True,
-    allow_methods=["*"],
-    allow_headers=["*"],
-)
-
-scraper: Optional[FIPIScraper] = None
-supabase_client: Optional[SupabaseClient] = None
-last_auto_refresh_at: Optional[datetime] = None
-refresh_in_progress = False
-
-
-@app.on_event("startup")
-async def startup_event() -> None:
-    global scraper, supabase_client
-
-    scraper = FIPIScraper(base_url=os.getenv("FIPI_BASE_URL", "https://fipi.ru"))
-    logger.info("FIPIScraper initialized")
-
-    if not SUPABASE_AVAILABLE:
-        logger.info("Supabase disabled")
-        return
-
-    supabase_url = os.getenv("SUPABASE_URL")
-    supabase_key = os.getenv("SUPABASE_SERVICE_KEY")
-    if not supabase_url or not supabase_key:
-        logger.warning("SUPABASE_URL or SUPABASE_SERVICE_KEY missing")
-        return
-
-    try:
-        client = SupabaseClient(url=supabase_url, key=supabase_key)
-        if await client.is_available():
-            supabase_client = client
-            logger.info("Supabase client connected")
-        else:
-            logger.error("Supabase is unavailable")
-    except Exception as exc:  # pragma: no cover - startup guard
-        logger.error("Supabase startup error: %s", exc)
-
-
-def _require_supabase() -> SupabaseClient:
-    if not supabase_client:
-        raise HTTPException(status_code=503, detail="Supabase is not configured")
-    return supabase_client
-
-
-def _require_scraper() -> FIPIScraper:
-    if not scraper:
-        raise HTTPException(status_code=503, detail="Scraper is not configured")
-    return scraper
-
-
-def _can_check_answer(task: Dict) -> bool:
-    if task.get("source_kind") == "dynamic_bank" and task.get("task_guid"):
-        return True
-
-    source_url = task.get("source_url", "")
-    parsed = urlparse(source_url)
-    query = parse_qs(parsed.query)
-    project_guid = (query.get("proj") or [None])[0]
-    question_id = (query.get("qid") or [None])[0]
-
-    return parsed.path.endswith("/questions.php") and bool(project_guid and (question_id or task.get("task_guid")))
-
-
-def _serialize_task(task: Dict) -> Dict:
-    payload = dict(task)
-    payload["can_check_answer"] = _can_check_answer(task)
-    return payload
-
-
-async def _persist_tasks(tasks: List[Dict]) -> Dict[str, int]:
-    client = _require_supabase()
-    saved = 0
-    duplicates = 0
-    for task in tasks:
-        result = await client.insert_task(task)
-        if result:
-            saved += 1
-        else:
-            duplicates += 1
-    return {"saved": saved, "duplicates": duplicates}
-
-
-async def _collect_tasks(subject: str = "russian", *, include_official_archives: bool = True) -> List[Dict]:
-    service = _require_scraper()
-    if include_official_archives:
-        return await service.scrape_tasks(subject=subject, include_official_archives=True)
-    return await service.scrape_dynamic_bank(subject=subject)
-
-
-async def _refresh_tasks(subject: str = "russian", *, include_official_archives: bool = True) -> Dict[str, int]:
-    scraped_tasks = await _collect_tasks(
-        subject=subject,
-        include_official_archives=include_official_archives,
-    )
-    persisted = await _persist_tasks(scraped_tasks)
-    return {
-        "scraped": len(scraped_tasks),
-        "saved": persisted["saved"],
-        "duplicates": persisted["duplicates"],
-    }
-
-
-def _needs_task_refresh(tasks: List[Dict]) -> bool:
-    if not tasks:
-        return True
-
-    dynamic_count = sum(1 for task in tasks if task.get("source_kind") == "dynamic_bank")
-    checkable_count = sum(1 for task in tasks if _can_check_answer(task))
-    minimum_total = max(10, int(os.getenv("SCRAPER_MIN_READY_TASKS", "40")))
-
-    if dynamic_count == 0 or checkable_count == 0:
-        return True
-
-    return len(tasks) < minimum_total
-
-
-def _needs_dynamic_bank_refresh(tasks: List[Dict]) -> bool:
-    if not tasks:
-        return True
-
-    dynamic_count = sum(1 for task in tasks if task.get("source_kind") == "dynamic_bank")
-    checkable_count = sum(1 for task in tasks if _can_check_answer(task))
-    return dynamic_count == 0 or checkable_count == 0
-
-
-def _is_refresh_running() -> bool:
-    return refresh_in_progress
-
-
-async def _run_refresh(subject: str = "russian", include_official_archives: bool = True) -> None:
-    global refresh_in_progress
-
-    try:
-        refresh_in_progress = True
-        refreshed = await _refresh_tasks(
-            subject=subject,
-            include_official_archives=include_official_archives,
-        )
-        logger.info(
-            "Background refresh finished: scraped=%s saved=%s duplicates=%s include_archives=%s",
-            refreshed["scraped"],
-            refreshed["saved"],
-            refreshed["duplicates"],
-            include_official_archives,
-        )
-    except Exception as exc:  # pragma: no cover - background guard
-        logger.error("Background refresh failed: %s", exc)
-    finally:
-        refresh_in_progress = False
-
-
-def _schedule_refresh(
-    background_tasks: BackgroundTasks,
-    subject: str = "russian",
-    *,
-    include_official_archives: bool = True,
-) -> bool:
-    global last_auto_refresh_at
-
-    if _is_refresh_running():
-        return False
-
-    last_auto_refresh_at = datetime.utcnow()
-    background_tasks.add_task(_run_refresh, subject, include_official_archives)
-    return True
-
-
-async def _ensure_tasks_available(background_tasks: BackgroundTasks, subject: str = "russian") -> List[Dict]:
-    global last_auto_refresh_at
-
-    client = _require_supabase()
-    existing = await client.get_all_tasks()
-    if existing and not _needs_task_refresh(existing):
-        return existing
-
-    if not existing:
-        logger.info("Tasks table is empty, running initial dynamic scrape")
-        last_auto_refresh_at = datetime.utcnow()
-        await _refresh_tasks(subject=subject, include_official_archives=False)
-        refreshed = await client.get_all_tasks()
-        if refreshed and _schedule_refresh(background_tasks, subject, include_official_archives=True):
-            logger.info("Scheduled full refresh after initial dynamic scrape")
-        return refreshed or existing
-
-    if _needs_dynamic_bank_refresh(existing):
-        if _is_refresh_running():
-            logger.info("Dynamic bank refresh is already running, returning existing tasks")
-            return existing
-
-        if _schedule_refresh(background_tasks, subject, include_official_archives=False):
-            logger.info("Tasks are missing dynamic/checkable entries, scheduled targeted dynamic refresh")
-        else:
-            logger.info("Unable to schedule targeted dynamic refresh, returning existing tasks")
-        return existing
-
-    cooldown_minutes = max(1, int(os.getenv("SCRAPER_AUTO_REFRESH_COOLDOWN_MINUTES", "30")))
-    if last_auto_refresh_at and (datetime.utcnow() - last_auto_refresh_at).total_seconds() < cooldown_minutes * 60:
-        logger.info("Skipping auto refresh because cooldown is active")
-        return existing
-
-    if _schedule_refresh(background_tasks, subject, include_official_archives=True):
-        logger.info("Existing tasks are stale or incomplete, scheduled background refresh")
-    else:
-        logger.info("Refresh is already running, returning existing tasks")
-
-    return existing
-
-
-def _normalize_answer(answer: str) -> str:
-    return re.sub(r"\s+", "", answer.strip()).upper()
-
-
-async def _resolve_task_guid(task: Dict) -> Optional[str]:
-    if task.get("task_guid"):
-        return task["task_guid"]
-
-    source_url = task.get("source_url", "")
-    if not _can_check_answer(task):
-        return None
-
-    html = await _require_scraper().fetch_page(source_url)
-    if not html:
-        return None
-
-    soup = BeautifulSoup(html, "lxml")
-    guid_input = soup.select_one("form[id^='checkform'] input[name='guid']")
-    return guid_input.get("value") if guid_input and guid_input.get("value") else None
-
-
-async def _check_task_answer(task: Dict, answer: str) -> CheckAnswerResponse:
-    if not _can_check_answer(task):
-        raise HTTPException(status_code=400, detail="This task does not support answer checking")
-
-    normalized = _normalize_answer(answer)
-    if not normalized:
-        raise HTTPException(status_code=400, detail="Answer is empty")
-
-    parsed = urlparse(task["source_url"])
-    query = parse_qs(parsed.query)
-    project_guid = (query.get("proj") or [None])[0]
-    task_guid = await _resolve_task_guid(task)
-
-    if not project_guid or not task_guid:
-        raise HTTPException(status_code=500, detail="Unable to resolve FIPI task metadata")
-
-    solve_url = f"{parsed.scheme}://{parsed.netloc}{parsed.path.rsplit('/', 1)[0]}/solve.php"
-    ssl_context = ssl.create_default_context()
-    ssl_context.check_hostname = False
-    ssl_context.verify_mode = ssl.CERT_NONE
-
-    async with httpx.AsyncClient(
-        headers=_require_scraper().headers,
-        timeout=45.0,
-        verify=ssl_context,
-        follow_redirects=True,
-    ) as client:
-        page_response = await client.get(task["source_url"])
-        page_response.raise_for_status()
-        response = await client.post(
-            solve_url,
-            data={
-                "guid": task_guid,
-                "answer": normalized,
-                "ajax": "1",
-                "proj": project_guid,
-            },
-            headers={"Referer": task["source_url"]},
-        )
-        response.raise_for_status()
-
-    if not response:
-        raise HTTPException(status_code=502, detail="FIPI answer check failed")
-
-    status_code = response.text.strip()
-    status_map = {
-        "0": ("not_solved", False, "Не решено"),
-        "1": ("solved", True, "Решено"),
-        "2": ("incorrect", False, "Неверно"),
-        "3": ("correct", True, "Верно"),
-    }
-    if status_code not in status_map:
-        raise HTTPException(status_code=502, detail=f"Unexpected FIPI response: {status_code}")
-
-    status_label, is_correct, message = status_map[status_code]
-    return CheckAnswerResponse(
-        success=True,
-        is_correct=is_correct,
-        status_code=status_label,
-        status_label=message,
-        submitted_answer=answer,
-        normalized_answer=normalized,
-        message=message,
-    )
-
-
-@app.get("/api/health", response_model=HealthResponse)
-async def health_check() -> HealthResponse:
-    services = {
-        "api": True,
-        "scraper": scraper is not None,
-        "supabase": False,
-    }
-
-    if supabase_client:
-        try:
-            services["supabase"] = await supabase_client.is_available()
-        except Exception:
-            services["supabase"] = False
-
-    all_critical_ok = services["api"] and services["scraper"]
-    if all_critical_ok and all(services.values()):
-        status = "healthy"
-    elif all_critical_ok:
-        status = "degraded"
-    else:
-        status = "unhealthy"
-
-    return HealthResponse(status=status, timestamp=datetime.utcnow(), services=services)
-
-
-@app.get("/api/tasks", response_model=List[TaskResponse])
-async def get_all_tasks(background_tasks: BackgroundTasks) -> List[TaskResponse]:
-    tasks = await _ensure_tasks_available(background_tasks)
-    return [TaskResponse(**_serialize_task(task)) for task in tasks]
-
-
-@app.get("/api/tasks/latest", response_model=List[TaskResponse])
-async def get_latest_tasks(limit: int = 10) -> List[TaskResponse]:
-    tasks = await _require_supabase().get_latest_tasks(limit=limit)
-    return [TaskResponse(**_serialize_task(task)) for task in tasks]
-
-
-@app.get("/api/tasks/{task_id}", response_model=TaskResponse)
-async def get_task(task_id: int) -> TaskResponse:
-    task = await _require_supabase().get_task_by_id(task_id)
-    if not task:
-        raise HTTPException(status_code=404, detail="Task not found")
-    return TaskResponse(**_serialize_task(task))
-
-
-@app.post("/api/tasks/{task_id}/check-answer", response_model=CheckAnswerResponse)
-async def check_task_answer(task_id: int, request: CheckAnswerRequest) -> CheckAnswerResponse:
-    task = await _require_supabase().get_task_by_id(task_id)
-    if not task:
-        raise HTTPException(status_code=404, detail="Task not found")
-    return await _check_task_answer(task, request.answer)
-
-
-@app.get("/api/tasks/type/{task_type}", response_model=List[TaskResponse])
-async def get_tasks_by_type(task_type: str) -> List[TaskResponse]:
-    tasks = await _require_supabase().get_tasks_by_type(task_type)
-    return [TaskResponse(**_serialize_task(task)) for task in tasks]
-
-
-@app.get("/api/tasks/search", response_model=List[TaskResponse])
-async def search_tasks(q: str) -> List[TaskResponse]:
-    tasks = await _require_supabase().search_tasks(q)
-    return [TaskResponse(**_serialize_task(task)) for task in tasks]
-
-
-@app.post("/api/scrape", response_model=ScrapeResponse)
-async def scrape_tasks(request: ScrapeRequest) -> ScrapeResponse:
-    client = _require_supabase()
-    service = _require_scraper()
-
-    try:
-        tasks_scraped = 0
-        tasks_saved = 0
-        duplicates_skipped = 0
-
-        if request.urls:
-            for url in request.urls:
-                html = await service.fetch_page(url)
-                task = service.parse_task_page(html, url) if html else None
-                if not task:
-                    continue
-                tasks_scraped += 1
-                result = await client.insert_task(task)
-                if result:
-                    tasks_saved += 1
-                else:
-                    duplicates_skipped += 1
-        elif request.query:
-            tasks = await service.search_tasks(request.query)
-            tasks_scraped = len(tasks)
-            persisted = await _persist_tasks(tasks)
-            tasks_saved = persisted["saved"]
-            duplicates_skipped = persisted["duplicates"]
-        else:
-            if request.full_refresh:
-                tasks = await service.scrape_tasks(
-                    subject=request.subject or "russian",
-                    include_official_archives=True,
-                )
-            else:
-                tasks = await service.scrape_dynamic_bank(subject=request.subject or "russian")
-            tasks_scraped = len(tasks)
-            persisted = await _persist_tasks(tasks)
-            tasks_saved = persisted["saved"]
-            duplicates_skipped = persisted["duplicates"]
-
-        return ScrapeResponse(
-            success=True,
-            tasks_scraped=tasks_scraped,
-            tasks_saved=tasks_saved,
-            duplicates_skipped=duplicates_skipped,
-            message=(
-                f"Processed {tasks_scraped} tasks. "
-                f"Saved: {tasks_saved}, duplicates: {duplicates_skipped}"
-            ),
-        )
-    except HTTPException:
-        raise
-    except Exception as exc:  # pragma: no cover - endpoint guard
-        logger.error("Scrape error: %s", exc)
-        raise HTTPException(status_code=500, detail=f"Scrape error: {exc}")
-
-
-@app.get("/api/stats", response_model=StatsResponse)
-async def get_stats() -> StatsResponse:
-    client = _require_supabase()
-    stats = await client.get_stats()
-    latest = await client.get_latest_tasks(limit=1)
-    last_scrape = latest[0].get("scraped_at") if latest else None
-    return StatsResponse(
-        total_tasks=stats.get("total", 0),
-        by_type=stats.get("by_type", {}),
-        last_scrape=last_scrape,
-    )
-
-
-@app.get("/", tags=["root"])
-async def root() -> Dict[str, str]:
-    return {
-        "message": "AI Scraper FIPI API proof-20260317",
-        "version": "1.1.1-proof-20260317",
-        "docs": "/docs",
-    }
-
-
-@app.exception_handler(Exception)
-async def global_exception_handler(request, exc) -> JSONResponse:
-    logger.error("Unhandled exception: %s", exc)
-    payload = ErrorResponse(
-        error="Internal Server Error",
-        detail=str(exc),
-        timestamp=datetime.utcnow(),
-    )
-    return JSONResponse(status_code=500, content=payload.model_dump(mode="json"))
-
-
-if __name__ == "__main__":
-    import uvicorn
-
-    uvicorn.run(app, host="0.0.0.0", port=8000)