feat(collectors): add Erasmus Mundus collector

Dockerfile

@@ -19,4 +19,4 @@ USER appuser
 
 EXPOSE 7860
 
-CMD ["sh", "-c", "alembic upgrade head && python -m scripts.check_campus_france && uvicorn app.main:app --host 0.0.0.0 --port ${PORT:-7860}"]
+CMD ["sh", "-c", "python -m scripts.check_erasmus_mundus_collector && uvicorn app.main:app --host 0.0.0.0 --port 7860"]

app/collectors/providers/erasmus_mundus.py

@@ -0,0 +1,1075 @@
+import asyncio
+import calendar
+import os
+import re
+from datetime import date, datetime
+from typing import Any
+from urllib.parse import urljoin, urlparse
+
+import feedparser
+import httpx
+from bs4 import BeautifulSoup
+
+from app.collectors.base import BaseOpportunityCollector
+from app.collectors.types import CollectedOpportunity
+
+
+class ErasmusMundusCollectorError(RuntimeError):
+    """Erreur produite pendant la collecte Erasmus Mundus."""
+
+
+class ErasmusMundusCollector(
+    BaseOpportunityCollector
+):
+    source_slug = "erasmus-mundus"
+
+    CATALOGUE_URL = (
+        "https://www.eacea.ec.europa.eu/"
+        "scholarships/"
+        "erasmus-mundus-catalogue_en"
+    )
+
+    RSS_URL = (
+        "https://www.eacea.ec.europa.eu/"
+        "node/253/rss_en"
+    )
+
+    REQUEST_TIMEOUT = 30.0
+
+    MAX_CONCURRENCY = 5
+
+    DEFAULT_MAX_PROGRAMMES = 60
+
+    APPLICATION_KEYWORDS = (
+        "apply",
+        "application",
+        "applications",
+        "admission",
+        "admissions",
+        "scholarship",
+        "scholarships",
+        "deadline",
+        "how-to-apply",
+        "how_to_apply",
+    )
+
+    BLOCKED_DOMAINS = (
+        "eacea.ec.europa.eu",
+        "erasmus-plus.ec.europa.eu",
+        "commission.europa.eu",
+        "education.ec.europa.eu",
+    )
+
+    MONTHS = {
+        "january": 1,
+        "february": 2,
+        "march": 3,
+        "april": 4,
+        "may": 5,
+        "june": 6,
+        "july": 7,
+        "august": 8,
+        "september": 9,
+        "october": 10,
+        "november": 11,
+        "december": 12,
+    }
+
+    DATE_PATTERNS = (
+        (
+            r"(?P<day>\d{1,2})(?:st|nd|rd|th)?"
+            r"\s+"
+            r"(?P<month>"
+            r"January|February|March|April|May|June|"
+            r"July|August|September|October|November|"
+            r"December"
+            r")"
+            r"\s*,?\s*"
+            r"(?P<year>20\d{2})"
+        ),
+        (
+            r"(?P<month>"
+            r"January|February|March|April|May|June|"
+            r"July|August|September|October|November|"
+            r"December"
+            r")"
+            r"\s+"
+            r"(?P<day>\d{1,2})(?:st|nd|rd|th)?"
+            r"\s*,?\s*"
+            r"(?P<year>20\d{2})"
+        ),
+        (
+            r"(?P<year>20\d{2})"
+            r"[-/.]"
+            r"(?P<month_number>\d{1,2})"
+            r"[-/.]"
+            r"(?P<day>\d{1,2})"
+        ),
+        (
+            r"(?P<day>\d{1,2})"
+            r"[-/.]"
+            r"(?P<month_number>\d{1,2})"
+            r"[-/.]"
+            r"(?P<year>20\d{2})"
+        ),
+    )
+
+    DEADLINE_CONTEXT_PATTERN = re.compile(
+        (
+            r"(?:"
+            r"application deadline|"
+            r"deadline for applications?|"
+            r"scholarship deadline|"
+            r"applications? close(?:s|d)?|"
+            r"applications? must be submitted by|"
+            r"submit(?:ted)? by|"
+            r"apply by|"
+            r"closing date"
+            r")"
+            r".{0,180}"
+        ),
+        flags=re.IGNORECASE,
+    )
+
+    SCHOLARSHIP_PATTERN = re.compile(
+        (
+            r"(?:"
+            r"erasmus mundus scholarship|"
+            r"erasmus\+ scholarship|"
+            r"full scholarship|"
+            r"fully funded|"
+            r"scholarship applications?|"
+            r"apply for (?:an? )?scholarship|"
+            r"scholarships? available"
+            r")"
+        ),
+        flags=re.IGNORECASE,
+    )
+
+    SCHOLARSHIP_UNAVAILABLE_PATTERN = re.compile(
+        (
+            r"(?:"
+            r"no scholarships? available|"
+            r"scholarships? (?:are|is) not available|"
+            r"without scholarship only|"
+            r"no erasmus mundus funding|"
+            r"self-funded students? only"
+            r")"
+        ),
+        flags=re.IGNORECASE,
+    )
+
+    def __init__(
+        self,
+        client: httpx.AsyncClient | None = None,
+    ) -> None:
+        self._external_client = client
+
+        configured_limit = os.getenv(
+            "ERASMUS_MUNDUS_MAX_PROGRAMMES"
+        )
+
+        try:
+            self.max_programmes = int(
+                configured_limit
+                or self.DEFAULT_MAX_PROGRAMMES
+            )
+        except ValueError:
+            self.max_programmes = (
+                self.DEFAULT_MAX_PROGRAMMES
+            )
+
+        self.max_programmes = max(
+            1,
+            min(self.max_programmes, 250),
+        )
+
+    async def collect(
+        self,
+    ) -> list[CollectedOpportunity]:
+        if self._external_client is not None:
+            return await self._collect_with_client(
+                self._external_client
+            )
+
+        headers = self._build_headers()
+
+        async with httpx.AsyncClient(
+            timeout=self.REQUEST_TIMEOUT,
+            follow_redirects=True,
+            headers=headers,
+        ) as client:
+            return await self._collect_with_client(
+                client
+            )
+
+    async def _collect_with_client(
+        self,
+        client: httpx.AsyncClient,
+    ) -> list[CollectedOpportunity]:
+        entries = await self._fetch_catalogue_entries(
+            client
+        )
+
+        semaphore = asyncio.Semaphore(
+            self.MAX_CONCURRENCY
+        )
+
+        async def process_entry(
+            entry: Any,
+        ) -> CollectedOpportunity | None:
+            async with semaphore:
+                try:
+                    return await self._process_entry(
+                        client=client,
+                        entry=entry,
+                    )
+                except (
+                    httpx.HTTPError,
+                    ErasmusMundusCollectorError,
+                    ValueError,
+                ):
+                    # Une défaillance sur un programme ne doit pas
+                    # empêcher la vérification des autres.
+                    return None
+
+        tasks = [
+            process_entry(entry)
+            for entry in entries[
+                : self.max_programmes
+            ]
+        ]
+
+        results = await asyncio.gather(*tasks)
+
+        opportunities = [
+            opportunity
+            for opportunity in results
+            if opportunity is not None
+        ]
+
+        opportunities.sort(
+            key=lambda opportunity: (
+                opportunity.deadline
+                or date.max
+            )
+        )
+
+        return opportunities
+
+    async def _fetch_catalogue_entries(
+        self,
+        client: httpx.AsyncClient,
+    ) -> list[Any]:
+        try:
+            response = await client.get(
+                self.RSS_URL,
+                headers=self._build_headers(),
+            )
+
+            response.raise_for_status()
+
+        except httpx.HTTPError as error:
+            raise ErasmusMundusCollectorError(
+                "Impossible de récupérer le flux RSS "
+                "officiel Erasmus Mundus."
+            ) from error
+
+        feed = await asyncio.to_thread(
+            feedparser.parse,
+            response.content,
+        )
+
+        entries = list(
+            getattr(feed, "entries", [])
+        )
+
+        if not entries:
+            raise ErasmusMundusCollectorError(
+                "Le catalogue Erasmus Mundus "
+                "n'a retourné aucun programme."
+            )
+
+        return entries
+
+    async def _process_entry(
+        self,
+        *,
+        client: httpx.AsyncClient,
+        entry: Any,
+    ) -> CollectedOpportunity | None:
+        title = self._clean_text(
+            getattr(entry, "title", "")
+        )
+
+        if not title:
+            return None
+
+        catalogue_url = self._extract_entry_url(
+            entry
+        )
+
+        if catalogue_url is None:
+            return None
+
+        programme_url = (
+            await self._find_programme_url(
+                client=client,
+                entry=entry,
+                catalogue_url=catalogue_url,
+            )
+        )
+
+        if programme_url is None:
+            return None
+
+        candidate_pages = (
+            await self._discover_application_pages(
+                client=client,
+                programme_url=programme_url,
+            )
+        )
+
+        for candidate_url in candidate_pages:
+            page = await self._fetch_page(
+                client=client,
+                url=candidate_url,
+            )
+
+            if page is None:
+                continue
+
+            html, final_url = page
+
+            page_text = self._extract_page_text(
+                html
+            )
+
+            if not self._scholarship_is_available(
+                page_text
+            ):
+                continue
+
+            deadline = self._extract_deadline(
+                page_text
+            )
+
+            if deadline is None:
+                continue
+
+            if deadline < date.today():
+                continue
+
+            description, image_url = (
+                self._extract_metadata(
+                    html=html,
+                    page_url=final_url,
+                )
+            )
+
+            summary = self._build_summary(
+                title=title,
+                deadline=deadline,
+                description=description,
+            )
+
+            publication_date = (
+                self._extract_publication_date(
+                    entry
+                )
+            )
+
+            return CollectedOpportunity(
+                source_slug=self.source_slug,
+                title=title[:250],
+                official_url=programme_url,
+                raw_content=page_text,
+                summary=summary,
+                description=description,
+                image_url=image_url,
+                organization_name=(
+                    "Erasmus Mundus"
+                ),
+                category="scholarship",
+                language="en",
+                publication_date=(
+                    publication_date
+                ),
+                deadline=deadline,
+                application_url=final_url,
+            )
+
+        return None
+
+    async def _find_programme_url(
+        self,
+        *,
+        client: httpx.AsyncClient,
+        entry: Any,
+        catalogue_url: str,
+    ) -> str | None:
+        entry_urls = self._extract_entry_urls(
+            entry
+        )
+
+        for url in entry_urls:
+            if self._is_external_programme_url(
+                url
+            ):
+                return url
+
+        page = await self._fetch_page(
+            client=client,
+            url=catalogue_url,
+        )
+
+        if page is None:
+            return None
+
+        html, final_url = page
+
+        soup = BeautifulSoup(
+            html,
+            "html.parser",
+        )
+
+        for anchor in soup.find_all(
+            "a",
+            href=True,
+        ):
+            href = anchor.get("href")
+
+            if not isinstance(href, str):
+                continue
+
+            candidate_url = urljoin(
+                final_url,
+                href.strip(),
+            )
+
+            if self._is_external_programme_url(
+                candidate_url
+            ):
+                return candidate_url
+
+        return None
+
+    async def _discover_application_pages(
+        self,
+        *,
+        client: httpx.AsyncClient,
+        programme_url: str,
+    ) -> list[str]:
+        page = await self._fetch_page(
+            client=client,
+            url=programme_url,
+        )
+
+        if page is None:
+            return [programme_url]
+
+        html, final_url = page
+
+        urls = [final_url]
+
+        soup = BeautifulSoup(
+            html,
+            "html.parser",
+        )
+
+        base_domain = self._normalized_domain(
+            final_url
+        )
+
+        for anchor in soup.find_all(
+            "a",
+            href=True,
+        ):
+            href = anchor.get("href")
+
+            if not isinstance(href, str):
+                continue
+
+            anchor_text = self._clean_text(
+                anchor.get_text(
+                    separator=" ",
+                    strip=True,
+                )
+            ).lower()
+
+            normalized_href = href.strip().lower()
+
+            searchable_value = (
+                f"{anchor_text} {normalized_href}"
+            )
+
+            if not any(
+                keyword in searchable_value
+                for keyword in (
+                    self.APPLICATION_KEYWORDS
+                )
+            ):
+                continue
+
+            candidate_url = urljoin(
+                final_url,
+                href.strip(),
+            )
+
+            parsed = urlparse(candidate_url)
+
+            if parsed.scheme not in {
+                "http",
+                "https",
+            }:
+                continue
+
+            if (
+                self._normalized_domain(
+                    candidate_url
+                )
+                != base_domain
+            ):
+                continue
+
+            if candidate_url not in urls:
+                urls.append(candidate_url)
+
+            if len(urls) >= 6:
+                break
+
+        return urls
+
+    async def _fetch_page(
+        self,
+        *,
+        client: httpx.AsyncClient,
+        url: str,
+    ) -> tuple[str, str] | None:
+        try:
+            response = await client.get(
+                url,
+                headers=self._build_headers(),
+            )
+
+            response.raise_for_status()
+
+        except httpx.HTTPError:
+            return None
+
+        content_type = response.headers.get(
+            "content-type",
+            "",
+        ).lower()
+
+        if "text/html" not in content_type:
+            return None
+
+        return response.text, str(response.url)
+
+    @classmethod
+    def _extract_deadline(
+        cls,
+        page_text: str,
+    ) -> date | None:
+        candidates: list[date] = []
+
+        for context_match in (
+            cls.DEADLINE_CONTEXT_PATTERN
+            .finditer(page_text)
+        ):
+            context_start = max(
+                0,
+                context_match.start() - 30,
+            )
+
+            context_end = min(
+                len(page_text),
+                context_match.end() + 220,
+            )
+
+            context = page_text[
+                context_start:context_end
+            ]
+
+            parsed_dates = (
+                cls._extract_dates_from_text(
+                    context
+                )
+            )
+
+            candidates.extend(parsed_dates)
+
+        future_dates = sorted(
+            {
+                candidate
+                for candidate in candidates
+                if candidate >= date.today()
+            }
+        )
+
+        if not future_dates:
+            return None
+
+        # La première échéance future associée
+        # explicitement à une candidature est retenue.
+        return future_dates[0]
+
+    @classmethod
+    def _extract_dates_from_text(
+        cls,
+        text: str,
+    ) -> list[date]:
+        parsed_dates: list[date] = []
+
+        for pattern in cls.DATE_PATTERNS:
+            for match in re.finditer(
+                pattern,
+                text,
+                flags=re.IGNORECASE,
+            ):
+                parsed_date = (
+                    cls._build_date_from_match(
+                        match
+                    )
+                )
+
+                if parsed_date is not None:
+                    parsed_dates.append(
+                        parsed_date
+                    )
+
+        return parsed_dates
+
+    @classmethod
+    def _build_date_from_match(
+        cls,
+        match: re.Match[str],
+    ) -> date | None:
+        groups = match.groupdict()
+
+        try:
+            day = int(groups["day"])
+            year = int(groups["year"])
+
+            month_name = groups.get(
+                "month"
+            )
+
+            if month_name:
+                month = cls.MONTHS[
+                    month_name.lower()
+                ]
+            else:
+                month = int(
+                    groups["month_number"]
+                )
+
+            return date(
+                year,
+                month,
+                day,
+            )
+
+        except (
+            KeyError,
+            TypeError,
+            ValueError,
+        ):
+            return None
+
+    @classmethod
+    def _scholarship_is_available(
+        cls,
+        page_text: str,
+    ) -> bool:
+        if cls.SCHOLARSHIP_UNAVAILABLE_PATTERN.search(
+            page_text
+        ):
+            return False
+
+        return (
+            cls.SCHOLARSHIP_PATTERN.search(
+                page_text
+            )
+            is not None
+        )
+
+    @classmethod
+    def _extract_metadata(
+        cls,
+        *,
+        html: str,
+        page_url: str,
+    ) -> tuple[str | None, str | None]:
+        soup = BeautifulSoup(
+            html,
+            "html.parser",
+        )
+
+        description_element = (
+            soup.find(
+                "meta",
+                attrs={
+                    "property": "og:description"
+                },
+            )
+            or soup.find(
+                "meta",
+                attrs={"name": "description"},
+            )
+            or soup.find(
+                "meta",
+                attrs={
+                    "name": "twitter:description"
+                },
+            )
+        )
+
+        image_element = (
+            soup.find(
+                "meta",
+                attrs={"property": "og:image"},
+            )
+            or soup.find(
+                "meta",
+                attrs={"name": "twitter:image"},
+            )
+        )
+
+        description: str | None = None
+        image_url: str | None = None
+
+        if description_element is not None:
+            content = description_element.get(
+                "content"
+            )
+
+            if isinstance(content, str):
+                normalized_description = (
+                    cls._clean_text(content)
+                )
+
+                if normalized_description:
+                    description = (
+                        normalized_description
+                    )
+
+        if image_element is not None:
+            content = image_element.get(
+                "content"
+            )
+
+            if isinstance(content, str):
+                normalized_url = (
+                    content.strip()
+                )
+
+                if normalized_url:
+                    image_url = urljoin(
+                        page_url,
+                        normalized_url,
+                    )
+
+        return description, image_url
+
+    @classmethod
+    def _extract_page_text(
+        cls,
+        html: str,
+    ) -> str:
+        soup = BeautifulSoup(
+            html,
+            "html.parser",
+        )
+
+        for element in soup(
+            [
+                "script",
+                "style",
+                "noscript",
+                "svg",
+                "header",
+                "footer",
+                "nav",
+            ]
+        ):
+            element.decompose()
+
+        main_content = (
+            soup.find("main")
+            or soup.find("article")
+            or soup.body
+        )
+
+        if main_content is None:
+            raise ErasmusMundusCollectorError(
+                "Le contenu principal du programme "
+                "Erasmus Mundus est introuvable."
+            )
+
+        page_text = main_content.get_text(
+            separator=" ",
+            strip=True,
+        )
+
+        normalized_text = cls._clean_text(
+            page_text
+        )
+
+        if len(normalized_text) < 100:
+            raise ErasmusMundusCollectorError(
+                "Le contenu Erasmus Mundus "
+                "extrait est anormalement court."
+            )
+
+        return normalized_text
+
+    @classmethod
+    def _build_summary(
+        cls,
+        *,
+        title: str,
+        deadline: date,
+        description: str | None,
+    ) -> str:
+        formatted_deadline = deadline.strftime(
+            "%d %B %Y"
+        )
+
+        if description:
+            summary = description
+        else:
+            summary = (
+                f"{title}. Erasmus Mundus Joint "
+                "Master scholarship programme."
+            )
+
+        summary = cls._clean_text(summary)
+
+        deadline_sentence = (
+            f" Application deadline: "
+            f"{formatted_deadline}."
+        )
+
+        available_length = (
+            500 - len(deadline_sentence)
+        )
+
+        trimmed_summary = summary[
+            :available_length
+        ].rstrip(" .")
+
+        return (
+            f"{trimmed_summary}."
+            f"{deadline_sentence}"
+        )[:500]
+
+    @staticmethod
+    def _extract_publication_date(
+        entry: Any,
+    ) -> date | None:
+        for attribute_name in (
+            "published_parsed",
+            "updated_parsed",
+        ):
+            parsed_value = getattr(
+                entry,
+                attribute_name,
+                None,
+            )
+
+            if parsed_value is None:
+                continue
+
+            try:
+                timestamp = calendar.timegm(
+                    parsed_value
+                )
+
+                return datetime.utcfromtimestamp(
+                    timestamp
+                ).date()
+
+            except (
+                TypeError,
+                ValueError,
+                OverflowError,
+            ):
+                continue
+
+        return None
+
+    @staticmethod
+    def _extract_entry_url(
+        entry: Any,
+    ) -> str | None:
+        url = getattr(
+            entry,
+            "link",
+            None,
+        )
+
+        if isinstance(url, str):
+            normalized_url = url.strip()
+
+            if normalized_url:
+                return normalized_url
+
+        return None
+
+    @classmethod
+    def _extract_entry_urls(
+        cls,
+        entry: Any,
+    ) -> list[str]:
+        urls: list[str] = []
+
+        entry_url = cls._extract_entry_url(
+            entry
+        )
+
+        if entry_url:
+            urls.append(entry_url)
+
+        links = getattr(
+            entry,
+            "links",
+            [],
+        )
+
+        for link in links:
+            href: str | None = None
+
+            if isinstance(link, dict):
+                value = link.get("href")
+
+                if isinstance(value, str):
+                    href = value
+
+            else:
+                value = getattr(
+                    link,
+                    "href",
+                    None,
+                )
+
+                if isinstance(value, str):
+                    href = value
+
+            if href:
+                normalized_url = href.strip()
+
+                if (
+                    normalized_url
+                    and normalized_url not in urls
+                ):
+                    urls.append(
+                        normalized_url
+                    )
+
+        summary = getattr(
+            entry,
+            "summary",
+            "",
+        )
+
+        if isinstance(summary, str):
+            soup = BeautifulSoup(
+                summary,
+                "html.parser",
+            )
+
+            for anchor in soup.find_all(
+                "a",
+                href=True,
+            ):
+                href = anchor.get("href")
+
+                if not isinstance(href, str):
+                    continue
+
+                normalized_url = href.strip()
+
+                if (
+                    normalized_url
+                    and normalized_url not in urls
+                ):
+                    urls.append(
+                        normalized_url
+                    )
+
+        return urls
+
+    @classmethod
+    def _is_external_programme_url(
+        cls,
+        url: str,
+    ) -> bool:
+        parsed = urlparse(url)
+
+        if parsed.scheme not in {
+            "http",
+            "https",
+        }:
+            return False
+
+        domain = cls._normalized_domain(
+            url
+        )
+
+        if not domain:
+            return False
+
+        return not any(
+            domain == blocked_domain
+            or domain.endswith(
+                f".{blocked_domain}"
+            )
+            for blocked_domain in (
+                cls.BLOCKED_DOMAINS
+            )
+        )
+
+    @staticmethod
+    def _normalized_domain(
+        url: str,
+    ) -> str:
+        domain = (
+            urlparse(url)
+            .netloc
+            .lower()
+            .split(":")[0]
+        )
+
+        if domain.startswith("www."):
+            domain = domain[4:]
+
+        return domain
+
+    @staticmethod
+    def _clean_text(
+        value: str,
+    ) -> str:
+        return re.sub(
+            r"\s+",
+            " ",
+            value,
+        ).strip()
+
+    @staticmethod
+    def _build_headers() -> dict[str, str]:
+        return {
+            "User-Agent": (
+                "VerifPulseOpportunityCollector/1.0 "
+                "(https://verifpulse-platform-zvit.vercel.app)"
+            ),
+            "Accept": (
+                "text/html,application/xhtml+xml,"
+                "application/rss+xml,"
+                "application/xml;q=0.9,*/*;q=0.8"
+            ),
+            "Accept-Language": (
+                "en-GB,en;q=0.9"
+            ),
+        }

app/collectors/registry.py

@@ -12,6 +12,9 @@ from app.collectors.providers.african_union import (
 from app.collectors.providers.campus_france import (
     CampusFranceCollector,
 )
+from app.collectors.providers.erasmus_mundus import (
+    ErasmusMundusCollector,
+)
 from app.collectors.providers.unesco import (
     UnescoCollector,
 )
@@ -42,4 +45,5 @@ def get_opportunity_collectors(
         MastercardFoundationCollector(),
         UnescoCollector(),
         CampusFranceCollector(),
+        ErasmusMundusCollector(),
     )

app/services/opportunity_sync.py

@@ -108,6 +108,22 @@ SOURCE_CONFIGURATION = {
         ),
         "source_tier": "A",
     },
+    "erasmus-mundus": {
+        "name": "Erasmus Mundus",
+        "base_url": (
+            "https://erasmus-plus.ec.europa.eu/"
+        ),
+        "opportunities_url": (
+            "https://www.eacea.ec.europa.eu/"
+            "scholarships/"
+            "erasmus-mundus-catalogue_en"
+        ),
+        "feed_url": (
+            "https://www.eacea.ec.europa.eu/"
+            "node/253/rss_en"
+        ),
+        "source_tier": "A",
+    },
 }
 
 

requirements.txt

@@ -6,4 +6,5 @@ psycopg[binary]>=3.2,<4.0
 alembic>=1.16,<2.0
 httpx>=0.28,<1.0
 beautifulsoup4>=4.13,<5.0
-pypdf>=3.17,<4.0
+pypdf>=3.17,<4.0
+feedparser>=6.0.11,<7.0.0