Delete scraper.py

scraper.py

@@ -1,877 +0,0 @@
-"""
-FIPI scraper focused on extracting real tasks instead of generic page text.
-"""
-
-from __future__ import annotations
-
-from datetime import datetime
-import io
-import logging
-import math
-import os
-import re
-import ssl
-from typing import Dict, Iterable, List, Optional
-from urllib.parse import urljoin
-import zipfile
-
-from bs4 import BeautifulSoup, Tag
-import httpx
-
-try:
-    from pypdf import PdfReader
-except ImportError:  # pragma: no cover - optional dependency for HF deploy
-    PdfReader = None
-
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger(__name__)
-
-
-class FIPIScraper:
-    """Collects task candidates from the FIPI bank and official demo archives."""
-
-    SUBJECT_CONFIG = {
-        "russian": {
-            "label": "Русский язык",
-            "dynamic_sources": [
-                {
-                    "kind": "ege_bank",
-                    "base_url": "https://ege.fipi.ru/bank",
-                    "project_guid": "AF0ED3F2557F8FFC4C06F80B6803FD26",
-                    "project_name": "ЕГЭ. Русский язык",
-                },
-                {
-                    "kind": "oge_bank",
-                    "base_url": "https://oge.fipi.ru/bank",
-                    "project_guid": "2F5EE3B12FE2A0EA40B06BF61A015416",
-                    "project_name": "ОГЭ. Русский язык",
-                },
-            ],
-            "official_demo_page": "https://fipi.ru/ege/demoversii-specifikacii-kodifikatory",
-            "official_variant_page": "https://fipi.ru/ege/otkrytyy-bank-zadaniy-ege/otkrytyye-varianty-kim-ege",
-            "archive_prefixes": ("ru_11_",),
-            "variant_prefixes": ("rus_",),
-            "title_keywords": ("русский язык",),
-        }
-    }
-
-    TASK_TYPE_KEYWORDS = {
-        "writing": ("сочинение", "эссе", "напишите", "сформулируйте", "прокомментируйте"),
-        "test": ("выберите", "укажите", "ответ", "вариант", "расставьте", "определите"),
-        "listening": ("аудио", "прослуш", "запись"),
-        "reading": ("прочитайте", "текст", "абзац", "предложение"),
-    }
-
-    GENERIC_TITLE_PATTERNS = (
-        "открытый банк",
-        "демоверсии",
-        "спецификации",
-        "кодификаторы",
-        "федеральный институт",
-        "фипи",
-        "нормативно",
-        "документы",
-        "варианты ким",
-    )
-
-    PDF_TASK_START_PATTERNS = (
-        "Прочитайте текст",
-        "Самостоятельно подберите",
-        "В тексте выделено",
-        "Укажите",
-        "В одном из",
-        "Отредактируйте предложение",
-        "Установите соответствие",
-        "Расставьте",
-        "Определите",
-        "Найдите",
-        "Подберите",
-    )
-
-    PDF_NOISE_PATTERNS = (
-        "Инструкция по выполнению работы",
-        "Пояснения к демонстрационному варианту",
-        "Желаем успеха",
-        "Все бланки ЕГЭ заполняются",
-        "Баллы, полученные",
-        "После завершения работы",
-        "В демонстрационном варианте представлены",
-        "Часть 1 содержит 26 заданий",
-        "На выполнение экзаменационной работы",
-        "Ответами к заданиям 1–26 являются",
-        "Бланк",
-    )
-
-    NOISE_PATTERNS = (
-        "федеральный институт педагогических измерений",
-        "открытый банк тестовых заданий",
-        "открытый банк заданий егэ",
-        "открытый банк заданий огэ",
-        "подбор заданий",
-        "демоверсии, спецификации, кодификаторы",
-        "для предметных комиссий",
-        "аналитические и методические материалы",
-        "видеоконсультации разработчиков ким",
-        "скачать",
-        "изменения в ким",
-    )
-
-    def __init__(self, base_url: str = "https://fipi.ru"):
-        self.base_url = base_url.rstrip("/")
-        self.headers = {
-            "User-Agent": (
-                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
-                "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
-            ),
-            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
-            "Accept-Language": "ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7",
-        }
-        self.page_size = max(1, int(os.getenv("SCRAPER_BANK_PAGE_SIZE", "10")))
-        self.max_bank_pages = max(1, int(os.getenv("SCRAPER_MAX_BANK_PAGES", "5")))
-        self.max_demo_archives = max(1, int(os.getenv("SCRAPER_MAX_DEMO_ARCHIVES", "2")))
-        self.max_demo_tasks = max(1, int(os.getenv("SCRAPER_MAX_DEMO_TASKS", "20")))
-        self.min_quality_score = max(1, int(os.getenv("SCRAPER_MIN_QUALITY_SCORE", "45")))
-
-    async def fetch_page(self, url: str) -> Optional[str]:
-        response = await self._request("GET", url)
-        return response.text if response else None
-
-    async def fetch_bytes(self, url: str) -> Optional[bytes]:
-        response = await self._request("GET", url)
-        return response.content if response else None
-
-    async def _request(
-        self,
-        method: str,
-        url: str,
-        *,
-        data: Optional[Dict[str, str]] = None,
-    ) -> Optional[httpx.Response]:
-        ssl_context = ssl.create_default_context()
-        ssl_context.check_hostname = False
-        ssl_context.verify_mode = ssl.CERT_NONE
-
-        async with httpx.AsyncClient(
-            headers=self.headers,
-            timeout=45.0,
-            verify=ssl_context,
-            follow_redirects=True,
-        ) as client:
-            try:
-                response = await client.request(method, url, data=data)
-                response.raise_for_status()
-                return response
-            except httpx.HTTPError as e:
-                logger.error("Request failed for %s: %s", url, e)
-                return None
-
-    async def scrape_tasks(
-        self,
-        subject: str = "russian",
-        *,
-        include_official_archives: bool = True,
-    ) -> List[Dict]:
-        config = self.SUBJECT_CONFIG.get(subject)
-        if not config:
-            logger.warning("Unknown subject %s, falling back to russian", subject)
-            config = self.SUBJECT_CONFIG["russian"]
-
-        candidates: List[Dict] = []
-        candidates.extend(await self.scrape_dynamic_bank(subject))
-        if include_official_archives:
-            candidates.extend(await self.scrape_official_archives(subject))
-        validated = self._dedupe_candidates(self._filter_candidates(candidates))
-        logger.info("Accepted %s task candidates after filtering", len(validated))
-        return validated
-
-    async def scrape_dynamic_bank(self, subject: str = "russian") -> List[Dict]:
-        config = self.SUBJECT_CONFIG.get(subject, self.SUBJECT_CONFIG["russian"])
-        tasks: List[Dict] = []
-
-        for source in config["dynamic_sources"]:
-            project_guid = source["project_guid"]
-            questions_url = f"{source['base_url']}/questions.php"
-            total_tasks = None
-
-            for page_index in range(self.max_bank_pages):
-                page_url = (
-                    f"{questions_url}?proj={project_guid}"
-                    f"&page={page_index}&pagesize={self.page_size}"
-                )
-                html = await self.fetch_page(page_url)
-                if not html and page_index == 0:
-                    html = await self._fetch_bank_first_page(questions_url, project_guid)
-                if not html:
-                    break
-
-                if total_tasks is None:
-                    total_tasks = self._extract_total_count(html)
-                    if total_tasks:
-                        max_pages = math.ceil(total_tasks / self.page_size)
-                        logger.info(
-                            "Bank %s reports %s tasks, scraping up to %s pages",
-                            source["project_name"],
-                            total_tasks,
-                            min(max_pages, self.max_bank_pages),
-                        )
-
-                soup = BeautifulSoup(html, "lxml")
-                blocks = soup.select("div.qblock")
-                if not blocks:
-                    break
-
-                for block in blocks:
-                    task = self._parse_bank_question_block(
-                        block,
-                        project_guid=project_guid,
-                        source_name=source["project_name"],
-                        questions_url=questions_url,
-                    )
-                    if task:
-                        tasks.append(task)
-
-                if total_tasks is not None and (page_index + 1) * self.page_size >= total_tasks:
-                    break
-
-        logger.info("Collected %s candidates from the dynamic bank", len(tasks))
-        return tasks
-
-    async def _fetch_bank_first_page(self, questions_url: str, project_guid: str) -> Optional[str]:
-        response = await self._request(
-            "POST",
-            questions_url,
-            data={
-                "search": "1",
-                "pagesize": str(self.page_size),
-                "proj": project_guid,
-            },
-        )
-        return response.text if response else None
-
-    def _extract_total_count(self, html: str) -> Optional[int]:
-        match = re.search(r"setQCount\((\d+)", html)
-        return int(match.group(1)) if match else None
-
-    def _parse_bank_question_block(
-        self,
-        block: Tag,
-        *,
-        project_guid: str,
-        source_name: str,
-        questions_url: str,
-    ) -> Optional[Dict]:
-        prompt_cell = block.select_one("td.cell_0")
-        if not prompt_cell:
-            return None
-
-        content = self._clean_text(prompt_cell.get_text("\n", strip=True))
-        if not content:
-            return None
-
-        title = self._build_title_from_content(content, fallback=source_name)
-        question_guid = self._extract_block_guid(block)
-        variants = self._extract_variants_from_block(block)
-        images = self._extract_images(prompt_cell, base_url=questions_url)
-
-        return {
-            "title": title,
-            "content": content,
-            "source_url": f"{questions_url}?proj={project_guid}&qid={question_guid}",
-            "task_type": self._detect_task_type(title, content),
-            "images": images,
-            "variants": variants,
-            "scraped_at": datetime.utcnow().isoformat(),
-            "source_kind": "dynamic_bank",
-            "task_guid": question_guid,
-        }
-
-    def _extract_block_guid(self, block: Tag) -> str:
-        guid_input = block.select_one("form input[name='guid']")
-        if guid_input and guid_input.get("value"):
-            return guid_input["value"]
-        return block.get("id", "").lstrip("q")
-
-    def _extract_variants_from_block(self, block: Tag) -> List[str]:
-        variants: List[str] = []
-
-        for label in block.find_all("label"):
-            text = self._clean_text(label.get_text(" ", strip=True))
-            if text:
-                variants.append(text)
-
-        if not variants:
-            for option in block.find_all("option"):
-                text = self._clean_text(option.get_text(" ", strip=True))
-                if text and text.lower() != "выбор":
-                    variants.append(text)
-
-        return variants[:10]
-
-    async def scrape_official_archives(self, subject: str = "russian") -> List[Dict]:
-        config = self.SUBJECT_CONFIG.get(subject, self.SUBJECT_CONFIG["russian"])
-        archive_links = await self._discover_official_archive_links(config)
-        variant_links = await self._discover_official_variant_links(config)
-        document_links = self._sort_document_links(archive_links + variant_links)
-        tasks: List[Dict] = []
-
-        if not document_links:
-            logger.warning("No official archive links found for %s", subject)
-            return tasks
-
-        if PdfReader is None:
-            logger.warning("pypdf is not installed, skipping official PDF extraction")
-            return tasks
-
-        for document_url in document_links[: self.max_demo_archives]:
-            document_bytes = await self.fetch_bytes(document_url)
-            if not document_bytes:
-                continue
-            tasks.extend(self._extract_tasks_from_document_bytes(document_bytes, document_url))
-
-        logger.info("Collected %s candidates from official archives", len(tasks))
-        return tasks
-
-    async def _discover_official_archive_links(self, config: Dict) -> List[str]:
-        html = await self.fetch_page(config["official_demo_page"])
-        if not html:
-            return []
-
-        soup = BeautifulSoup(html, "lxml")
-        prefixes = config["archive_prefixes"]
-        archive_links: List[str] = []
-
-        for link in soup.find_all("a", href=True):
-            href = link["href"]
-            absolute = href if href.startswith("http") else urljoin(config["official_demo_page"], href)
-            href_lower = absolute.lower()
-            if not href_lower.endswith(".zip"):
-                continue
-            if any(prefix in href_lower for prefix in prefixes):
-                archive_links.append(absolute)
-
-        def sort_key(url: str) -> int:
-            match = re.search(r"/(20\d{2})/", url)
-            return int(match.group(1)) if match else 0
-
-        archive_links.sort(key=sort_key, reverse=True)
-        return archive_links
-
-    async def _discover_official_variant_links(self, config: Dict) -> List[str]:
-        variant_page = config.get("official_variant_page")
-        if not variant_page:
-            return []
-
-        html = await self.fetch_page(variant_page)
-        if not html:
-            return []
-
-        soup = BeautifulSoup(html, "lxml")
-        prefixes = config.get("variant_prefixes", ())
-        links: List[str] = []
-
-        for link in soup.find_all("a", href=True):
-            href = link["href"]
-            absolute = href if href.startswith("http") else urljoin(variant_page, href)
-            href_lower = absolute.lower()
-            if not href_lower.endswith((".zip", ".pdf")):
-                continue
-            if "braille" in href_lower:
-                continue
-            filename = absolute.rsplit("/", 1)[-1].lower()
-            if prefixes and not any(filename.startswith(prefix) for prefix in prefixes):
-                continue
-            links.append(absolute)
-
-        return self._sort_document_links(links)
-
-    def _sort_document_links(self, links: Iterable[str]) -> List[str]:
-        def sort_key(url: str) -> tuple[int, str]:
-            match = re.search(r"(20\d{2})", url)
-            return (int(match.group(1)) if match else 0, url)
-
-        return sorted(set(links), key=sort_key, reverse=True)
-
-    def _extract_tasks_from_document_bytes(self, document_bytes: bytes, document_url: str) -> List[Dict]:
-        if document_url.lower().endswith(".zip"):
-            return self._extract_tasks_from_archive(document_bytes, document_url)
-        if document_url.lower().endswith(".pdf"):
-            return self._extract_tasks_from_pdf_document(
-                document_bytes,
-                document_url=document_url,
-                document_name=document_url.rsplit("/", 1)[-1],
-            )
-        return []
-
-    def _extract_tasks_from_archive(self, archive_bytes: bytes, archive_url: str) -> List[Dict]:
-        tasks: List[Dict] = []
-
-        try:
-            with zipfile.ZipFile(io.BytesIO(archive_bytes)) as archive:
-                for member_name in archive.namelist():
-                    if not member_name.lower().endswith(".pdf"):
-                        continue
-                    if "демо" not in member_name.lower() and "demo" not in member_name.lower():
-                        continue
-
-                    text = self._extract_text_from_pdf_bytes(archive.read(member_name))
-                    if not text:
-                        continue
-
-                    year_match = re.search(r"(20\d{2})", archive_url)
-                    year = year_match.group(1) if year_match else "unknown"
-                    tasks.extend(
-                        self._extract_tasks_from_demo_text(
-                            text,
-                            archive_url=archive_url,
-                            document_name=member_name,
-                            year=year,
-                        )
-                    )
-        except zipfile.BadZipFile:
-            logger.error("Invalid archive %s", archive_url)
-
-        return tasks
-
-    def _extract_text_from_pdf_bytes(self, pdf_bytes: bytes) -> str:
-        if PdfReader is None:
-            return ""
-
-        try:
-            reader = PdfReader(io.BytesIO(pdf_bytes))
-        except Exception as e:  # pragma: no cover - parser-dependent
-            logger.error("Failed to open PDF: %s", e)
-            return ""
-
-        pages: List[str] = []
-        for page in reader.pages:
-            try:
-                page_text = page.extract_text() or ""
-            except Exception:  # pragma: no cover - parser-dependent
-                page_text = ""
-            if page_text:
-                pages.append(page_text)
-
-        return self._clean_text("\n".join(pages))
-
-    def _extract_tasks_from_demo_text(
-        self,
-        text: str,
-        *,
-        archive_url: str,
-        document_name: str,
-        year: str,
-    ) -> List[Dict]:
-        tasks: List[Dict] = []
-        if not text:
-            return tasks
-
-        bounded_text = text
-        if not bounded_text:
-            return tasks
-
-        pattern = re.compile(
-            r"(?ms)(?:^|\n)(\d{1,2})[\.\)]\s*(.+?)(?=(?:\n\d{1,2}[\.\)])|(?:\nЧасть\s+\d)|\Z)"
-        )
-
-        for match in pattern.finditer(bounded_text):
-            task_number = int(match.group(1))
-            content = self._clean_text(match.group(2))
-            if len(content) < 80:
-                continue
-
-            title = f"Демоверсия ЕГЭ {year}. Задание {task_number}"
-            tasks.append(
-                {
-                    "title": title,
-                    "content": content,
-                    "source_url": f"{archive_url}#task-{task_number}",
-                    "task_type": self._detect_task_type(title, content),
-                    "images": [],
-                    "variants": self._extract_variants(content),
-                    "scraped_at": datetime.utcnow().isoformat(),
-                    "source_kind": "official_demo_pdf",
-                    "document_name": document_name,
-                    "task_number": task_number,
-                }
-            )
-
-            if len(tasks) >= self.max_demo_tasks:
-                break
-
-        return tasks
-
-    def _slice_demo_section(self, text: str) -> str:
-        start = re.search(r"(Часть\s*1|Ответами к заданиям)", text, re.IGNORECASE)
-        if not start:
-            return text
-
-        end = re.search(r"(Система оценивания|Ключи|Ответы)", text[start.start() :], re.IGNORECASE)
-        if not end:
-            return text[start.start() :]
-
-        return text[start.start() : start.start() + end.start()]
-
-    def _extract_tasks_from_archive(self, archive_bytes: bytes, archive_url: str) -> List[Dict]:
-        tasks: List[Dict] = []
-
-        try:
-            with zipfile.ZipFile(io.BytesIO(archive_bytes)) as archive:
-                for member_name in archive.namelist():
-                    if not member_name.lower().endswith(".pdf"):
-                        continue
-                    if not self._should_parse_pdf_member(member_name, archive_url):
-                        continue
-                    tasks.extend(
-                        self._extract_tasks_from_pdf_document(
-                            archive.read(member_name),
-                            document_url=archive_url,
-                            document_name=member_name,
-                        )
-                    )
-        except zipfile.BadZipFile:
-            logger.error("Invalid archive %s", archive_url)
-
-        return tasks
-
-    def _should_parse_pdf_member(self, member_name: str, document_url: str) -> bool:
-        member_lower = member_name.lower()
-        if any(token in member_lower for token in ("спец", "кодиф", "критер", "ответ", "аудио")):
-            return False
-        if "otkrytyye-varianty-kim-ege" in document_url.lower():
-            return True
-        return "демо" in member_lower or "demo" in member_lower
-
-    def _extract_tasks_from_pdf_document(
-        self,
-        pdf_bytes: bytes,
-        *,
-        document_url: str,
-        document_name: str,
-    ) -> List[Dict]:
-        text = self._extract_text_from_pdf_bytes(pdf_bytes)
-        if not text:
-            return []
-
-        year_match = re.search(r"(20\d{2})", document_url)
-        year = year_match.group(1) if year_match else "unknown"
-        return self._extract_tasks_from_demo_text(
-            text,
-            archive_url=document_url,
-            document_name=document_name,
-            year=year,
-            source_kind=self._detect_document_source_kind(document_url),
-        )
-
-    def _detect_document_source_kind(self, document_url: str) -> str:
-        if "otkrytyye-varianty-kim-ege" in document_url.lower():
-            return "official_open_variant_pdf"
-        return "official_demo_pdf"
-
-    def _extract_tasks_from_demo_text(
-        self,
-        text: str,
-        *,
-        archive_url: str,
-        document_name: str,
-        year: str,
-        source_kind: str = "official_demo_pdf",
-    ) -> List[Dict]:
-        tasks: List[Dict] = []
-        if not text:
-            return tasks
-
-        bounded_text = text
-        if not bounded_text:
-            return tasks
-
-        for raw_block in self._split_pdf_into_task_blocks(bounded_text):
-            content = self._cleanup_pdf_task_block(raw_block)
-            content = self._trim_to_task_start(content)
-            if not self._looks_like_official_task_block(content):
-                continue
-
-            task_number = len(tasks) + 1
-            document_label = "Открытый вариант ЕГЭ" if source_kind == "official_open_variant_pdf" else "Демоверсия ЕГЭ"
-            title = f"{document_label} {year}. Задание {task_number}"
-            tasks.append(
-                {
-                    "title": title,
-                    "content": content,
-                    "source_url": f"{archive_url}#task-{task_number}",
-                    "task_type": self._detect_task_type(title, content),
-                    "images": [],
-                    "variants": self._extract_variants(content),
-                    "scraped_at": datetime.utcnow().isoformat(),
-                    "source_kind": source_kind,
-                    "document_name": document_name,
-                    "task_number": task_number,
-                }
-            )
-
-            if len(tasks) >= self.max_demo_tasks:
-                break
-
-        return tasks
-
-    def _split_pdf_into_task_blocks(self, text: str) -> List[str]:
-        answer_pattern = re.compile(r"(?:^|\n)\s*Ответ\s*:\s*[_\.\s]*", re.IGNORECASE)
-        blocks: List[str] = []
-        last_pos = 0
-
-        for match in answer_pattern.finditer(text):
-            block = text[last_pos:match.start()]
-            if block.strip():
-                blocks.append(block)
-            last_pos = match.end()
-
-        return blocks
-
-    def _cleanup_pdf_task_block(self, block: str) -> str:
-        lines: List[str] = []
-        for raw_line in block.splitlines():
-            line = self._clean_text(raw_line)
-            if not line:
-                continue
-            lower = line.lower()
-            if line == "&%end_page&%":
-                continue
-            if re.fullmatch(r"\d{1,2}", line):
-                continue
-            if re.search(r"\d+\s*/\s*\d+$", line):
-                continue
-            if lower.startswith(("демонстрационный вариант егэ", "открытый вариант ким егэ", "единый государственный экзамен")):
-                continue
-            if lower.startswith("© "):
-                continue
-            lines.append(line)
-
-        return self._clean_text("\n".join(lines))
-
-    def _trim_to_task_start(self, text: str) -> str:
-        if not text:
-            return text
-
-        starts = [text.find(pattern) for pattern in self.PDF_TASK_START_PATTERNS if text.find(pattern) >= 0]
-        if starts:
-            return text[min(starts):].strip()
-        return text.strip()
-
-    def _looks_like_official_task_block(self, text: str) -> bool:
-        if len(text) < 70 or len(text) > 6000:
-            return False
-
-        lower = text.lower()
-        if any(pattern.lower() in lower for pattern in self.PDF_NOISE_PATTERNS):
-            return False
-
-        return any(pattern.lower() in lower for pattern in self.PDF_TASK_START_PATTERNS)
-
-    def _slice_demo_section(self, text: str) -> str:
-        start_matches = list(re.finditer(r"(?m)^\s*Часть\s*1\s*$", text, re.IGNORECASE))
-        if start_matches:
-            start_pos = start_matches[-1].start()
-        else:
-            fallback = list(re.finditer(r"Ответами к заданиям", text, re.IGNORECASE))
-            if not fallback:
-                return text
-            start_pos = fallback[-1].start()
-
-        end = re.search(
-            r"(Часть\s*2|Задание\s*27|Система оценивания|Критерии оценивания|Ключи)",
-            text[start_pos:],
-            re.IGNORECASE,
-        )
-        if not end:
-            return text[start_pos:]
-
-        return text[start_pos : start_pos + end.start()]
-
-    def parse_task_page(self, html: str, url: str) -> Optional[Dict]:
-        if not html:
-            return None
-
-        soup = BeautifulSoup(html, "lxml")
-        for selector in (
-            "div.qblock",
-            "article",
-            "main article",
-            ".field--name-body",
-            ".content",
-            "main",
-            "body",
-        ):
-            container = soup.select_one(selector)
-            if not container:
-                continue
-
-            candidate = self._build_candidate_from_container(container, url)
-            if candidate:
-                return candidate
-
-        return None
-
-    def _build_candidate_from_container(self, container: Tag, url: str) -> Optional[Dict]:
-        cloned = BeautifulSoup(str(container), "lxml")
-        root = cloned.find()
-        if root is None:
-            return None
-
-        for element in root.find_all(["script", "style", "nav", "header", "footer", "form", "button", "aside"]):
-            element.decompose()
-
-        title_tag = root.find(["h1", "h2", "h3", "strong", "b"])
-        title = self._clean_text(title_tag.get_text(" ", strip=True)) if title_tag else ""
-        content = self._clean_text(root.get_text("\n", strip=True))
-        if not title:
-            title = self._build_title_from_content(content, fallback=url)
-
-        images = self._extract_images(root, base_url=url)
-        candidate = {
-            "title": title,
-            "content": content,
-            "source_url": url,
-            "task_type": self._detect_task_type(title, content),
-            "images": images,
-            "variants": self._extract_variants(content),
-            "scraped_at": datetime.utcnow().isoformat(),
-            "source_kind": "generic_html",
-        }
-        return candidate if self._passes_quality_gate(candidate) else None
-
-    async def scrape_task_by_id(self, task_id: str) -> Optional[Dict]:
-        config = self.SUBJECT_CONFIG["russian"]["dynamic_sources"][0]
-        html = await self.fetch_page(
-            f"{config['base_url']}/questions.php?proj={config['project_guid']}&qid={task_id}"
-        )
-        if not html:
-            return None
-
-        soup = BeautifulSoup(html, "lxml")
-        block = soup.select_one("div.qblock")
-        if not block:
-            return None
-
-        return self._parse_bank_question_block(
-            block,
-            project_guid=config["project_guid"],
-            source_name=config["project_name"],
-            questions_url=f"{config['base_url']}/questions.php",
-        )
-
-    async def search_tasks(self, query: str) -> List[Dict]:
-        query_lower = query.lower().strip()
-        tasks = await self.scrape_tasks(subject="russian")
-        return [
-            task
-            for task in tasks
-            if query_lower in task.get("title", "").lower()
-            or query_lower in task.get("content", "").lower()
-        ]
-
-    def _filter_candidates(self, candidates: Iterable[Dict]) -> List[Dict]:
-        accepted: List[Dict] = []
-        for candidate in candidates:
-            if self._passes_quality_gate(candidate):
-                accepted.append(candidate)
-        return accepted
-
-    def _dedupe_candidates(self, candidates: Iterable[Dict]) -> List[Dict]:
-        deduped: List[Dict] = []
-        seen_keys = set()
-
-        for candidate in candidates:
-            normalized = self._clean_text(candidate.get("content", ""))[:400]
-            key = (candidate.get("source_url", ""), normalized)
-            if key in seen_keys:
-                continue
-            seen_keys.add(key)
-            deduped.append(candidate)
-
-        return deduped
-
-    def _passes_quality_gate(self, candidate: Dict) -> bool:
-        score = self._score_candidate(candidate)
-        candidate["quality_score"] = score
-        return score >= self.min_quality_score
-
-    def _score_candidate(self, candidate: Dict) -> int:
-        title = candidate.get("title", "").lower()
-        content = candidate.get("content", "").lower()
-        source_kind = candidate.get("source_kind", "")
-        length = len(content)
-
-        score = 0
-
-        if source_kind == "dynamic_bank":
-            score += 60
-        elif source_kind in {"official_demo_pdf", "official_open_variant_pdf"}:
-            score += 50
-        else:
-            score += 10
-
-        if 80 <= length <= 3500:
-            score += 15
-        elif length > 5000:
-            score -= 20
-        else:
-            score -= 10
-
-        if any(keyword in content for keywords in self.TASK_TYPE_KEYWORDS.values() for keyword in keywords):
-            score += 10
-
-        if any(pattern.lower() in content for pattern in self.PDF_TASK_START_PATTERNS):
-            score += 10
-
-        if re.search(r"\b\d+\b", content):
-            score += 5
-
-        if any(pattern in title for pattern in self.GENERIC_TITLE_PATTERNS):
-            score -= 45
-
-        noise_hits = sum(1 for pattern in self.NOISE_PATTERNS if pattern in content[:1200])
-        score -= min(noise_hits * 8, 32)
-
-        if content.count("\n") > 80:
-            score -= 10
-
-        return score
-
-    def _detect_task_type(self, title: str, content: str) -> str:
-        text = f"{title} {content}".lower()
-
-        for task_type, keywords in self.TASK_TYPE_KEYWORDS.items():
-            if any(keyword in text for keyword in keywords):
-                return task_type
-
-        return "other"
-
-    def _extract_variants(self, content: str) -> List[str]:
-        matches = re.findall(r"(?:^|\n)(?:[1-6]|[A-DА-Г])[.)]\s*([^\n]{2,200})", content)
-        return [self._clean_text(match) for match in matches[:10]]
-
-    def _extract_images(self, container: Tag, *, base_url: str) -> List[str]:
-        images: List[str] = []
-        for img in container.find_all("img"):
-            src = img.get("src") or img.get("data-src")
-            if not src:
-                continue
-            images.append(src if src.startswith("http") else urljoin(base_url, src))
-        return images[:10]
-
-    def _build_title_from_content(self, content: str, fallback: str) -> str:
-        first_line = next((line.strip() for line in content.splitlines() if line.strip()), "")
-        title = self._clean_text(first_line)
-        if not title:
-            title = fallback
-        return title[:160]
-
-    def _clean_text(self, text: str) -> str:
-        text = text.replace("\xa0", " ")
-        text = re.sub(
-            r"\b(?:[A-Za-zА-Яа-яЁё]\s+){2,}[A-Za-zА-Яа-яЁё]\b",
-            lambda match: match.group(0).replace(" ", ""),
-            text,
-        )
-        text = re.sub(r"[ \t]+", " ", text)
-        text = re.sub(r"\n{3,}", "\n\n", text)
-        return text.strip()