feat(collectors): add African Union opportunities collector

Dockerfile

@@ -19,4 +19,4 @@ USER appuser
 
 EXPOSE 7860
 
-CMD ["sh", "-c", "alembic upgrade head && python -m scripts.check_university_bamenda_collector && uvicorn app.main:app --host 0.0.0.0 --port 7860"]
+CMD ["sh", "-c", "alembic upgrade head && python -m scripts.check_african_union_collector && uvicorn app.main:app --host 0.0.0.0 --port 7860"]

app/collectors/providers/african_union.py

@@ -0,0 +1,807 @@
+from __future__ import annotations
+
+import asyncio
+import re
+from dataclasses import dataclass
+from datetime import date, datetime, timedelta
+from urllib.parse import parse_qsl, urlencode, urljoin, urlparse, urlunparse
+
+import httpx
+from bs4 import BeautifulSoup
+
+from app.collectors.base import BaseOpportunityCollector
+from app.collectors.types import CollectedOpportunity
+
+
+class AfricanUnionCollectorError(RuntimeError):
+    """Raised when an official African Union resource cannot be read."""
+
+
+@dataclass(frozen=True, slots=True)
+class Candidate:
+    title: str
+    url: str
+    channel: str
+    publication_date: date | None = None
+
+
+class AfricanUnionCollector(BaseOpportunityCollector):
+    """Collect current jobs, internships and programme opportunities."""
+
+    source_slug = "african-union"
+
+    BASE_URL = "https://au.int/"
+    JOBS_URL = "https://jobs.au.int/"
+    JOB_SEARCH_URLS = (
+        "https://jobs.au.int/",
+        "https://jobs.au.int/search/",
+        (
+            "https://jobs.au.int/search/"
+            "?createNewAlert=false&q=&locationsearch="
+        ),
+    )
+    ANNOUNCEMENT_URLS = (
+        "https://au.int/en/announcements",
+        "https://au.int/en/pressreleases",
+    )
+
+    ALLOWED_HOSTS = {
+        "au.int",
+        "www.au.int",
+        "jobs.au.int",
+        "careers.au.int",
+        "career2.successfactors.eu",
+    }
+
+    OPPORTUNITY_PATTERNS = (
+        r"\binternship(?:s)?\b",
+        r"\bfellowship(?:s)?\b",
+        r"\bscholarship(?:s)?\b",
+        r"\bvacanc(?:y|ies)\b",
+        r"\bjob(?:s)?\b",
+        r"\bcareer(?:s)?\b",
+        r"\bcall\s+for\s+(?:applications?|candidates?|proposals?)\b",
+        r"\bapplications?\s+(?:are\s+)?open\b",
+        r"\byouth\s+volunteer\b",
+        r"\bvolunteer(?:ing|s)?\b",
+        r"\btraineeship(?:s)?\b",
+        r"\bcompetition(?:s)?\b",
+        r"\bprogramme?\b",
+    )
+
+    CLOSED_PATTERNS = (
+        r"\bapplications?\s+(?:are\s+)?closed\b",
+        r"\bcall\s+(?:is\s+)?closed\b",
+        r"\bdeadline\s+has\s+passed\b",
+        r"\bno\s+longer\s+accepting\b",
+        r"\bexpired\b",
+    )
+
+    RESULT_PATTERNS = (
+        r"\bselected\s+candidates?\b",
+        r"\bsuccessful\s+candidates?\b",
+        r"\bselection\s+results?\b",
+        r"\bresults?\s+of\s+the\s+call\b",
+        r"\bcohort\s+announced\b",
+    )
+
+    MONTHS = {
+        "january": 1,
+        "february": 2,
+        "march": 3,
+        "april": 4,
+        "may": 5,
+        "june": 6,
+        "july": 7,
+        "august": 8,
+        "september": 9,
+        "october": 10,
+        "november": 11,
+        "december": 12,
+        "jan": 1,
+        "feb": 2,
+        "mar": 3,
+        "apr": 4,
+        "jun": 6,
+        "jul": 7,
+        "aug": 8,
+        "sep": 9,
+        "sept": 9,
+        "oct": 10,
+        "nov": 11,
+        "dec": 12,
+    }
+
+    COUNTRY_CODES = {
+        "algeria": "DZ",
+        "cameroon": "CM",
+        "ethiopia": "ET",
+        "gambia": "GM",
+        "ghana": "GH",
+        "kenya": "KE",
+        "morocco": "MA",
+        "nigeria": "NG",
+        "rwanda": "RW",
+        "senegal": "SN",
+        "south africa": "ZA",
+        "tanzania": "TZ",
+        "tunisia": "TN",
+        "uganda": "UG",
+        "zambia": "ZM",
+        "zimbabwe": "ZW",
+    }
+
+    MAX_ARTICLE_AGE = timedelta(days=180)
+
+    def __init__(self) -> None:
+        self._client = httpx.AsyncClient(
+            timeout=httpx.Timeout(45.0, connect=15.0),
+            follow_redirects=True,
+            headers={
+                "User-Agent": (
+                    "Mozilla/5.0 (compatible; VerifPulseBot/1.0; "
+                    "+https://verifpulse-platform.vercel.app)"
+                ),
+                "Accept-Language": "en,fr;q=0.8",
+            },
+        )
+        self._semaphore = asyncio.Semaphore(3)
+
+    async def collect(self) -> list[CollectedOpportunity]:
+        try:
+            candidates = await self._discover_candidates()
+            print(
+                f"[African Union] {len(candidates)} unique candidate(s) "
+                "discovered.",
+                flush=True,
+            )
+
+            results = await asyncio.gather(
+                *(self._inspect_candidate(candidate) for candidate in candidates),
+                return_exceptions=True,
+            )
+
+            opportunities: list[CollectedOpportunity] = []
+            for candidate, result in zip(candidates, results):
+                if isinstance(result, Exception):
+                    print(
+                        "[African Union] Candidate ignored: "
+                        f"{candidate.url} - {type(result).__name__}: {result}",
+                        flush=True,
+                    )
+                elif result is not None:
+                    opportunities.append(result)
+
+            opportunities = self._deduplicate(opportunities)
+            opportunities.sort(
+                key=lambda item: (
+                    item.deadline is not None,
+                    item.deadline or date.max,
+                    item.title.casefold(),
+                )
+            )
+
+            print(
+                f"[African Union] {len(opportunities)} current "
+                "opportunity/opportunities retained.",
+                flush=True,
+            )
+            return opportunities
+        finally:
+            await self._client.aclose()
+
+    async def _discover_candidates(self) -> list[Candidate]:
+        jobs, announcements = await asyncio.gather(
+            self._discover_job_candidates(),
+            self._discover_announcement_candidates(),
+        )
+
+        unique: dict[str, Candidate] = {}
+        for candidate in (*jobs, *announcements):
+            key = self._candidate_key(candidate)
+            unique.setdefault(key, candidate)
+        return list(unique.values())
+
+    async def _discover_job_candidates(self) -> list[Candidate]:
+        discovered: list[Candidate] = []
+
+        for listing_url in self.JOB_SEARCH_URLS:
+            try:
+                response = await self._get(listing_url)
+            except AfricanUnionCollectorError as error:
+                print(
+                    f"[African Union] Jobs listing ignored: {listing_url} - {error}",
+                    flush=True,
+                )
+                continue
+
+            soup = BeautifulSoup(response.text, "html.parser")
+            count_before = len(discovered)
+
+            for link in soup.select("a[href]"):
+                url = self._canonical_url(urljoin(str(response.url), link["href"]))
+                if urlparse(url).hostname != "jobs.au.int" or "/job/" not in url:
+                    continue
+
+                title = self._clean_text(link.get_text(" ", strip=True))
+                if not title or title.casefold() in {"view job", "view details"}:
+                    title = self._title_from_job_url(url)
+
+                discovered.append(Candidate(title=title, url=url, channel="jobs"))
+
+            print(
+                "[African Union] "
+                f"{len(discovered) - count_before} job candidate(s) found on "
+                f"{listing_url}",
+                flush=True,
+            )
+
+        return discovered
+
+    async def _discover_announcement_candidates(self) -> list[Candidate]:
+        discovered: list[Candidate] = []
+
+        for listing_url in self.ANNOUNCEMENT_URLS:
+            try:
+                response = await self._get(listing_url)
+            except AfricanUnionCollectorError as error:
+                print(
+                    "[African Union] Announcement listing ignored: "
+                    f"{listing_url} - {error}",
+                    flush=True,
+                )
+                continue
+
+            soup = BeautifulSoup(response.text, "html.parser")
+            count_before = len(discovered)
+
+            for link in soup.select("a[href]"):
+                title = self._clean_text(link.get_text(" ", strip=True))
+                if not title or not self._matches(title, self.OPPORTUNITY_PATTERNS):
+                    continue
+                if self._matches(title, self.RESULT_PATTERNS):
+                    continue
+
+                url = self._canonical_url(urljoin(str(response.url), link["href"]))
+                parsed = urlparse(url)
+                if parsed.hostname not in {"au.int", "www.au.int"}:
+                    continue
+                if not any(
+                    section in parsed.path
+                    for section in ("/announcements/", "/pressreleases/")
+                ):
+                    continue
+
+                publication_date = self._extract_date_from_context(link)
+                discovered.append(
+                    Candidate(
+                        title=title,
+                        url=url,
+                        channel="announcement",
+                        publication_date=publication_date,
+                    )
+                )
+
+            print(
+                "[African Union] "
+                f"{len(discovered) - count_before} programme candidate(s) found "
+                f"on {listing_url}",
+                flush=True,
+            )
+
+        return discovered
+
+    async def _inspect_candidate(
+        self,
+        candidate: Candidate,
+    ) -> CollectedOpportunity | None:
+        response = await self._get(candidate.url)
+        soup = BeautifulSoup(response.text, "html.parser")
+
+        if candidate.channel == "jobs":
+            return self._parse_job(candidate, soup, str(response.url))
+        return self._parse_announcement(candidate, soup, str(response.url))
+
+    def _parse_job(
+        self,
+        candidate: Candidate,
+        soup: BeautifulSoup,
+        final_url: str,
+    ) -> CollectedOpportunity | None:
+        text = self._main_text(soup)
+        normalized = text.casefold()
+
+        if self._matches(normalized, self.CLOSED_PATTERNS):
+            return None
+
+        req_id = self._field(text, "Req Id")
+        title = self._field(text, "Job Title") or candidate.title
+        employee_class = self._field(text, "Employee Class")
+        location = self._field(text, "Job Location (country)")
+        department = self._field(text, "Department")
+        au_organ = self._field(text, "AU Organ")
+
+        publication_date = self._parse_date(
+            self._field(text, "Posting Start Date")
+        )
+        deadline = self._parse_date(self._field(text, "End Date"))
+        if deadline is None:
+            deadline = self._extract_deadline(text)
+
+        if deadline is not None and deadline < date.today():
+            return None
+
+        category = self._job_category(title, employee_class)
+        summary = self._job_summary(soup, text, title)
+        application_url = self._extract_apply_url(soup, final_url)
+        country_code = self.COUNTRY_CODES.get((location or "").casefold())
+
+        requires_manual_review = deadline is None
+        if requires_manual_review and publication_date:
+            if date.today() - publication_date > self.MAX_ARTICLE_AGE:
+                return None
+
+        eligibility_text = self._extract_section(
+            text,
+            ("Who can Apply?", "Eligibility"),
+            ("How to apply?", "Application process", "Selection process"),
+        )
+
+        return CollectedOpportunity(
+            source_slug=self.source_slug,
+            title=title,
+            official_url=self._canonical_url(final_url),
+            summary=summary,
+            description=self._truncate(text, 8000),
+            organization_name=au_organ or "African Union Commission",
+            category=category,
+            country_code=country_code,
+            location=location,
+            language="en",
+            publication_date=publication_date,
+            deadline=deadline,
+            application_url=application_url,
+            image_url=None,
+            target_countries=(),
+            study_levels=self._study_levels(text),
+            eligibility={
+                "deadline_status": (
+                    "confirmed" if deadline else "not_extracted"
+                ),
+                "requires_manual_review": requires_manual_review,
+                "scope": "African Union member states",
+                "req_id": req_id,
+                "employee_class": employee_class,
+                "department": department,
+                "au_organ": au_organ,
+                "eligibility_text": self._truncate(eligibility_text, 2000),
+            },
+            raw_content=text,
+        )
+
+    def _parse_announcement(
+        self,
+        candidate: Candidate,
+        soup: BeautifulSoup,
+        final_url: str,
+    ) -> CollectedOpportunity | None:
+        title = self._article_title(soup) or candidate.title
+        text = self._article_text(soup)
+        normalized = f"{title} {text}".casefold()
+
+        if not self._matches(normalized, self.OPPORTUNITY_PATTERNS):
+            return None
+        if self._matches(normalized, self.CLOSED_PATTERNS):
+            return None
+        if self._matches(normalized, self.RESULT_PATTERNS):
+            return None
+
+        publication_date = (
+            self._extract_publication_date(soup)
+            or candidate.publication_date
+        )
+        deadline = self._extract_deadline(text)
+        if deadline is not None and deadline < date.today():
+            return None
+
+        requires_manual_review = deadline is None
+        if requires_manual_review:
+            if publication_date is None:
+                return None
+            if date.today() - publication_date > self.MAX_ARTICLE_AGE:
+                return None
+
+        documents = self._official_documents(soup, final_url)
+        application_url = self._extract_application_url(
+            soup,
+            final_url,
+            documents,
+        )
+        category = self._programme_category(normalized)
+
+        return CollectedOpportunity(
+            source_slug=self.source_slug,
+            title=title,
+            official_url=self._canonical_url(final_url),
+            summary=self._summary(text, title, deadline),
+            description=self._truncate(text, 8000),
+            organization_name="African Union Commission",
+            category=category,
+            country_code=None,
+            location=None,
+            language=self._detect_language(text),
+            publication_date=publication_date,
+            deadline=deadline,
+            application_url=application_url,
+            image_url=self._article_image(soup, final_url),
+            target_countries=(),
+            study_levels=self._study_levels(text),
+            eligibility={
+                "deadline_status": (
+                    "confirmed" if deadline else "not_extracted"
+                ),
+                "requires_manual_review": requires_manual_review,
+                "scope": "African Union member states",
+                "article_url": self._canonical_url(final_url),
+                "official_documents": documents,
+            },
+            raw_content=text,
+        )
+
+    async def _get(self, url: str) -> httpx.Response:
+        self._assert_allowed_url(url)
+        async with self._semaphore:
+            try:
+                response = await self._client.get(url)
+                response.raise_for_status()
+            except httpx.HTTPError as error:
+                raise AfricanUnionCollectorError(
+                    f"Unable to fetch African Union resource: {url}"
+                ) from error
+
+        self._assert_allowed_url(str(response.url))
+        return response
+
+    def _assert_allowed_url(self, url: str) -> None:
+        parsed = urlparse(url)
+        if parsed.scheme != "https" or parsed.hostname not in self.ALLOWED_HOSTS:
+            raise AfricanUnionCollectorError(
+                f"Non-official African Union URL rejected: {url}"
+            )
+
+    @staticmethod
+    def _clean_text(value: str) -> str:
+        return re.sub(r"\s+", " ", value).strip()
+
+    @classmethod
+    def _main_text(cls, soup: BeautifulSoup) -> str:
+        root = soup.select_one("main, #job-content, .job, article") or soup.body
+        return cls._clean_text(root.get_text(" ", strip=True) if root else "")
+
+    @classmethod
+    def _article_text(cls, soup: BeautifulSoup) -> str:
+        root = soup.select_one(
+            "article .field--name-body, article .content, article, main"
+        ) or soup.body
+        if root is None:
+            return ""
+        clone = BeautifulSoup(str(root), "html.parser")
+        for node in clone.select(
+            "nav, header, footer, script, style, .social-share, .related-content"
+        ):
+            node.decompose()
+        return cls._clean_text(clone.get_text(" ", strip=True))
+
+    @classmethod
+    def _article_title(cls, soup: BeautifulSoup) -> str | None:
+        node = soup.select_one("h1, article h1, .page-title")
+        return cls._clean_text(node.get_text(" ", strip=True)) if node else None
+
+    @classmethod
+    def _field(cls, text: str, label: str) -> str | None:
+        labels = (
+            "Req Id",
+            "Job Title",
+            "Posting Start Date",
+            "Employee Class",
+            "End Date",
+            "Job Location (country)",
+            "Department",
+            "AU Organ",
+        )
+        next_labels = "|".join(re.escape(item) for item in labels if item != label)
+        match = re.search(
+            rf"{re.escape(label)}\s*:\s*(.*?)\s*(?=(?:{next_labels})\s*:|$)",
+            text,
+            flags=re.IGNORECASE,
+        )
+        value = cls._clean_text(match.group(1)) if match else ""
+        return value or None
+
+    @classmethod
+    def _parse_date(cls, value: str | None) -> date | None:
+        if not value:
+            return None
+        cleaned = cls._clean_text(value).replace(",", " ")
+
+        numeric = re.search(r"\b(\d{1,2})/(\d{1,2})/(\d{2,4})\b", cleaned)
+        if numeric:
+            month, day, year = (int(part) for part in numeric.groups())
+            year += 2000 if year < 100 else 0
+            try:
+                return date(year, month, day)
+            except ValueError:
+                return None
+
+        textual = re.search(
+            r"\b([A-Za-z]+)\s+(\d{1,2})(?:st|nd|rd|th)?\s+(\d{4})\b",
+            cleaned,
+            flags=re.IGNORECASE,
+        )
+        if textual:
+            month_name, day, year = textual.groups()
+            month = cls.MONTHS.get(month_name.casefold())
+            if month:
+                try:
+                    return date(int(year), month, int(day))
+                except ValueError:
+                    return None
+
+        reverse = re.search(
+            r"\b(\d{1,2})(?:st|nd|rd|th)?\s+([A-Za-z]+)\s+(\d{4})\b",
+            cleaned,
+            flags=re.IGNORECASE,
+        )
+        if reverse:
+            day, month_name, year = reverse.groups()
+            month = cls.MONTHS.get(month_name.casefold())
+            if month:
+                try:
+                    return date(int(year), month, int(day))
+                except ValueError:
+                    return None
+        return None
+
+    @classmethod
+    def _extract_deadline(cls, text: str) -> date | None:
+        patterns = (
+            r"(?:applications? must be submitted no later than|"
+            r"application deadline|deadline for applications?|closing date|"
+            r"deadline)\s*(?:is|:|-)?\s*"
+            r"([A-Za-z]+\s+\d{1,2}(?:st|nd|rd|th)?[,]?\s+\d{4})",
+            r"(?:applications? must be submitted no later than|"
+            r"application deadline|deadline for applications?|closing date|"
+            r"deadline)\s*(?:is|:|-)?\s*"
+            r"(\d{1,2}\s+[A-Za-z]+\s+\d{4})",
+            r"(?:end date)\s*(?:is|:|-)?\s*"
+            r"([A-Za-z]+\s+\d{1,2}[,]?\s+\d{4})",
+        )
+        for pattern in patterns:
+            match = re.search(pattern, text, flags=re.IGNORECASE)
+            if match:
+                parsed = cls._parse_date(match.group(1))
+                if parsed:
+                    return parsed
+        return None
+
+    @classmethod
+    def _extract_publication_date(cls, soup: BeautifulSoup) -> date | None:
+        for selector in (
+            "meta[property='article:published_time']",
+            "meta[name='date']",
+            "time[datetime]",
+        ):
+            node = soup.select_one(selector)
+            if not node:
+                continue
+            value = node.get("content") or node.get("datetime") or node.get_text(" ")
+            if not value:
+                continue
+            try:
+                return datetime.fromisoformat(str(value).replace("Z", "+00:00")).date()
+            except ValueError:
+                parsed = cls._parse_date(str(value))
+                if parsed:
+                    return parsed
+        return None
+
+    @classmethod
+    def _extract_date_from_context(cls, link) -> date | None:
+        parent = link.find_parent(["article", "li", "div"])
+        text = parent.get_text(" ", strip=True) if parent else ""
+        return cls._parse_date(text)
+
+    @classmethod
+    def _job_category(cls, title: str, employee_class: str | None) -> str:
+        value = f"{title} {employee_class or ''}".casefold()
+        if "intern" in value:
+            return "internship"
+        if "fellow" in value:
+            return "fellowship"
+        return "job"
+
+    @classmethod
+    def _programme_category(cls, text: str) -> str:
+        if "internship" in text or "traineeship" in text:
+            return "internship"
+        if "scholarship" in text:
+            return "scholarship"
+        if "fellowship" in text:
+            return "fellowship"
+        if "volunteer" in text:
+            return "program"
+        if "competition" in text:
+            return "competition"
+        if "vacancy" in text or re.search(r"\bjob\b", text):
+            return "job"
+        return "program"
+
+    @classmethod
+    def _study_levels(cls, text: str) -> tuple[str, ...]:
+        normalized = text.casefold()
+        levels: list[str] = []
+        mappings = (
+            ("bachelor", ("bachelor", "undergraduate", "first degree")),
+            ("master", ("master", "postgraduate")),
+            ("phd", ("phd", "doctoral", "doctorate")),
+        )
+        for level, markers in mappings:
+            if any(marker in normalized for marker in markers):
+                levels.append(level)
+        return tuple(levels)
+
+    @classmethod
+    def _job_summary(cls, soup: BeautifulSoup, text: str, title: str) -> str:
+        section = cls._extract_section(
+            text,
+            ("Job Description", "Background", "Purpose of Job"),
+            ("Context", "Main Functions", "Who can Apply?", "Eligibility"),
+        )
+        if not section:
+            section = text
+        return cls._summary(section, title, None)
+
+    @classmethod
+    def _summary(cls, text: str, title: str, deadline: date | None) -> str:
+        cleaned = cls._clean_text(text)
+        if cleaned.casefold().startswith(title.casefold()):
+            cleaned = cleaned[len(title):].lstrip(" :-")
+        sentence_match = re.search(r"^(.{80,460}?[.!?])(?:\s|$)", cleaned)
+        summary = sentence_match.group(1) if sentence_match else cleaned[:460]
+        if not summary:
+            summary = f"Official African Union opportunity: {title}."
+        if deadline is None:
+            suffix = " Consult the official page for the closing date."
+            summary = summary[: 500 - len(suffix)].rstrip() + suffix
+        return summary[:500]
+
+    @classmethod
+    def _extract_section(
+        cls,
+        text: str,
+        starts: tuple[str, ...],
+        ends: tuple[str, ...],
+    ) -> str:
+        start_positions = [
+            position
+            for marker in starts
+            if (position := text.casefold().find(marker.casefold())) >= 0
+        ]
+        if not start_positions:
+            return ""
+        start = min(start_positions)
+        end_positions = [
+            position
+            for marker in ends
+            if (position := text.casefold().find(marker.casefold(), start + 1)) >= 0
+        ]
+        end = min(end_positions) if end_positions else min(len(text), start + 2500)
+        return cls._clean_text(text[start:end])
+
+    @classmethod
+    def _extract_apply_url(cls, soup: BeautifulSoup, fallback: str) -> str:
+        for link in soup.select("a[href]"):
+            label = cls._clean_text(link.get_text(" ", strip=True)).casefold()
+            if label in {"i'm interested", "apply", "apply now"}:
+                return cls._canonical_url(urljoin(fallback, link["href"]))
+        return cls._canonical_url(fallback)
+
+    @classmethod
+    def _extract_application_url(
+        cls,
+        soup: BeautifulSoup,
+        fallback: str,
+        documents: list[str],
+    ) -> str:
+        for link in soup.select("a[href]"):
+            label = cls._clean_text(link.get_text(" ", strip=True)).casefold()
+            if re.search(r"\b(apply|application form|register|submit)\b", label):
+                url = cls._canonical_url(urljoin(fallback, link["href"]))
+                if urlparse(url).scheme == "https":
+                    return url
+        return documents[0] if documents else cls._canonical_url(fallback)
+
+    @classmethod
+    def _official_documents(cls, soup: BeautifulSoup, base_url: str) -> list[str]:
+        documents: list[str] = []
+        for link in soup.select("a[href]"):
+            url = cls._canonical_url(urljoin(base_url, link["href"]))
+            if urlparse(url).path.casefold().endswith((".pdf", ".doc", ".docx")):
+                documents.append(url)
+        return list(dict.fromkeys(documents))
+
+    @classmethod
+    def _article_image(cls, soup: BeautifulSoup, base_url: str) -> str | None:
+        node = soup.select_one("meta[property='og:image'], article img[src]")
+        if not node:
+            return None
+        value = node.get("content") or node.get("src")
+        return urljoin(base_url, str(value)) if value else None
+
+    @staticmethod
+    def _detect_language(text: str) -> str:
+        french_markers = ("appel à", "candidature", "date limite", "bourse")
+        normalized = text.casefold()
+        return "fr" if sum(marker in normalized for marker in french_markers) >= 2 else "en"
+
+    @staticmethod
+    def _matches(text: str, patterns: tuple[str, ...]) -> bool:
+        return any(re.search(pattern, text, flags=re.IGNORECASE) for pattern in patterns)
+
+    @classmethod
+    def _title_from_job_url(cls, url: str) -> str:
+        path = urlparse(url).path
+        match = re.search(r"/job/([^/]+)/", path)
+        value = match.group(1) if match else "African Union opportunity"
+        return cls._clean_text(value.replace("%2C", ",").replace("-", " "))
+
+    @staticmethod
+    def _truncate(value: str, limit: int) -> str:
+        return value if len(value) <= limit else value[: limit - 1].rstrip() + "…"
+
+    @classmethod
+    def _canonical_url(cls, url: str) -> str:
+        parsed = urlparse(url)
+        query = [
+            (key, value)
+            for key, value in parse_qsl(parsed.query, keep_blank_values=True)
+            if not key.casefold().startswith("utm_")
+            and key.casefold() not in {"locale", "source", "ref"}
+        ]
+        return urlunparse(
+            (
+                parsed.scheme.casefold(),
+                parsed.netloc.casefold(),
+                re.sub(r"/{2,}", "/", parsed.path).rstrip("/") or "/",
+                "",
+                urlencode(query),
+                "",
+            )
+        )
+
+    @classmethod
+    def _candidate_key(cls, candidate: Candidate) -> str:
+        req_id = re.search(r"/(\d+)(?:-[^/]*)?$", urlparse(candidate.url).path)
+        if candidate.channel == "jobs" and req_id:
+            return f"job:{req_id.group(1)}"
+        return cls._canonical_url(candidate.url)
+
+    @classmethod
+    def _deduplicate(
+        cls,
+        opportunities: list[CollectedOpportunity],
+    ) -> list[CollectedOpportunity]:
+        unique: dict[str, CollectedOpportunity] = {}
+        for opportunity in opportunities:
+            req_id = opportunity.eligibility.get("req_id")
+            key = (
+                f"job:{req_id}"
+                if req_id
+                else cls._canonical_url(opportunity.official_url)
+            )
+            existing = unique.get(key)
+            if existing is None or (
+                existing.deadline is None and opportunity.deadline is not None
+            ):
+                unique[key] = opportunity
+        return list(unique.values())

app/services/opportunity_sync.py

@@ -93,6 +93,12 @@ SOURCE_CONFIGURATION = {
         ),
         "source_tier": "A",
     },
+    "african-union": {
+        "name": "African Union",
+        "base_url": "https://au.int/",
+        "opportunities_url": "https://jobs.au.int/",
+        "source_tier": "A",
+    },
 }
 
 

scripts/check_african_union_collector.py

@@ -0,0 +1,65 @@
+import asyncio
+import json
+import sys
+from dataclasses import asdict
+from datetime import date
+from typing import Any
+
+from app.collectors.providers.african_union import (
+    AfricanUnionCollector,
+    AfricanUnionCollectorError,
+)
+
+
+def serialize_value(value: Any) -> Any:
+    if isinstance(value, date):
+        return value.isoformat()
+    if isinstance(value, (list, tuple)):
+        return [serialize_value(item) for item in value]
+    if isinstance(value, dict):
+        return {key: serialize_value(item) for key, item in value.items()}
+    return value
+
+
+async def main() -> None:
+    collector = AfricanUnionCollector()
+    print(
+        "[collector-check] Starting African Union opportunities check...",
+        flush=True,
+    )
+
+    opportunities = await collector.collect()
+    if not opportunities:
+        raise RuntimeError("African Union returned no current opportunity.")
+
+    for opportunity in opportunities:
+        payload = serialize_value(asdict(opportunity))
+        raw_content = payload.pop("raw_content", "")
+        payload["raw_content_length"] = len(raw_content)
+        print(json.dumps(payload, indent=2, ensure_ascii=False), flush=True)
+
+    print(
+        "[collector-check] African Union check passed: "
+        f"{len(opportunities)} opportunity/opportunities collected.",
+        flush=True,
+    )
+
+
+if __name__ == "__main__":
+    try:
+        asyncio.run(main())
+    except AfricanUnionCollectorError as error:
+        print(
+            f"[collector-check] Collection failed: {error}",
+            file=sys.stderr,
+            flush=True,
+        )
+        raise SystemExit(1) from error
+    except Exception as error:
+        print(
+            "[collector-check] Unexpected failure: "
+            f"{type(error).__name__}: {error}",
+            file=sys.stderr,
+            flush=True,
+        )
+        raise SystemExit(1) from error